Publications

@misc{shimabucoro2025posttrainersguidemultilingualtraining,
  title = {A Post-trainer's Guide to Multilingual Training Data: Uncovering Cross-lingual Transfer Dynamics},
  author = {Shimabucoro, Luisa and Ustun, Ahmet and Fadaee, Marzieh and Ruder, Sebastian},
  year = {2025},
  eprint = {2504.16677},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

ArXiv

Déjà Vu: Multilingual LLM Evaluation through the Lens of Machine Translation Evaluation

Julia Kreutzer, Eleftheria Briakou, Sweta Agrawal, Marzieh Fadaee, and Kocmi Tom

2025

@misc{kreutzer2025dejavumultilingualllm,
  title = {D\'ej\`a Vu: Multilingual LLM Evaluation through the Lens of Machine Translation Evaluation},
  author = {Kreutzer, Julia and Briakou, Eleftheria and Agrawal, Sweta and Fadaee, Marzieh and Tom, Kocmi},
  year = {2025},
  eprint = {2504.11829},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

ArXiv

Kaleidoscope: In-language Exams for Massively Multilingual Vision Evaluation

Israfel Salazar, Manuel Fernández Burda, Shayekh Bin Islam, Arshia Soltani Moakhar, Shivalika Singh, Fabian Farestam, Angelika Romanou, Danylo Boiko, Dipika Khullar, Mike Zhang, Dominik Krzemiński, Jekaterina Novikova, Luísa Shimabucoro, Joseph Marvin Imperial, Rishabh Maheshwary, Sharad Duwal, Alfonso Amayuelas, Swati Rajwal, Jebish Purbey, Ahmed Ruby, Nicholas Popovič, Marek Suppa, Azmine Toushik Wasi, Ram Mohan Rao Kadiyala, Olga Tsymboi, Maksim Kostritsya, Bardia Soltani Moakhar, Gabriel Costa Merlin, Otávio Ferracioli Coletti, Maral Jabbari Shiviari, MohammadAmin fard, Silvia Fernandez, María Grandury, Dmitry Abulkhanov, Drishti Sharma, Andre Guarnier De Mitri, Leticia Bossatto Marchezi, Setayesh Heydari, Johan Obando-Ceron, Nazar Kohut, Beyza Ermis, Desmond Elliott, Enzo Ferrante, Sara Hooker, and Marzieh Fadaee

2025

@misc{salazar2025kaleidoscopeinlanguageexamsmassively,
  title = {Kaleidoscope: In-language Exams for Massively Multilingual Vision Evaluation},
  author = {Salazar, Israfel and Burda, Manuel Fernández and Islam, Shayekh Bin and Moakhar, Arshia Soltani and Singh, Shivalika and Farestam, Fabian and Romanou, Angelika and Boiko, Danylo and Khullar, Dipika and Zhang, Mike and Krzemiński, Dominik and Novikova, Jekaterina and Shimabucoro, Luísa and Imperial, Joseph Marvin and Maheshwary, Rishabh and Duwal, Sharad and Amayuelas, Alfonso and Rajwal, Swati and Purbey, Jebish and Ruby, Ahmed and Popovič, Nicholas and Suppa, Marek and Wasi, Azmine Toushik and Kadiyala, Ram Mohan Rao and Tsymboi, Olga and Kostritsya, Maksim and Moakhar, Bardia Soltani and da Costa Merlin, Gabriel and Coletti, Otávio Ferracioli and Shiviari, Maral Jabbari and farahani fard, MohammadAmin and Fernandez, Silvia and Grandury, María and Abulkhanov, Dmitry and Sharma, Drishti and Mitri, Andre Guarnier De and Marchezi, Leticia Bossatto and Heydari, Setayesh and Obando-Ceron, Johan and Kohut, Nazar and Ermis, Beyza and Elliott, Desmond and Ferrante, Enzo and Hooker, Sara and Fadaee, Marzieh},
  year = {2025},
  eprint = {2504.07072},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

ArXiv

Command A: An Enterprise-Ready Large Language Model

Team Cohere, Aakanksha, Arash Ahmadian, Marwan Ahmed, Jay Alammar, Milad Alizadeh, Yazeed Alnumay, Sophia Althammer, Arkady Arkhangorodsky, Viraat Aryabumi, Dennis Aumiller, Raphaël Avalos, Zahara Aviv, Sammie Bae, Saurabh Baji, Alexandre Barbet, Max Bartolo, Björn Bebensee, Neeral Beladia, Walter Beller-Morales, Alexandre Bérard, Andrew Berneshawi, Anna Bialas, Phil Blunsom, Matt Bobkin, Adi Bongale, Sam Braun, Maxime Brunet, Samuel Cahyawijaya, David Cairuz, Jon Ander Campos, Cassie Cao, Kris Cao, Roman Castagné, Julián Cendrero, Leila Chan Currie, Yash Chandak, Diane Chang, Giannis Chatziveroglou, Hongyu Chen, Claire Cheng, Alexis Chevalier, Justin T. Chiu, Eugene Cho, Eugene Choi, Eujeong Choi, Tim Chung, Volkan Cirik, Ana Cismaru, Pierre Clavier, Henry Conklin, Lucas Crawhall-Stein, Devon Crouse, Andres Felipe Cruz-Salinas, Ben Cyrus, Daniel D’souza, Hugo Dalla-Torre, John Dang, William Darling, Omar Darwiche Domingues, Saurabh Dash, Antoine Debugne, Théo Dehaze, Shaan Desai, Joan Devassy, Rishit Dholakia, Kyle Duffy, Ali Edalati, Ace Eldeib, Abdullah Elkady, Sarah Elsharkawy, Irem Ergün, Beyza Ermis, Marzieh Fadaee, Boyu Fan, Lucas Fayoux, Yannis Flet-Berliac, Nick Frosst, Matthias Gallé, Wojciech Galuba, Utsav Garg, Matthieu Geist, Mohammad Gheshlaghi Azar, Ellen Gilsenan-McMahon, Seraphina Goldfarb-Tarrant, Tomas Goldsack, Aidan Gomez, Victor Machado Gonzaga, Nithya Govindarajan, Manoj Govindassamy, Nathan Grinsztajn, Nikolas Gritsch, Patrick Gu, Shangmin Guo, Kilian Haefeli, Rod Hajjar, Tim Hawes, Jingyi He, Sebastian Hofstätter, Sungjin Hong, Sara Hooker, Tom Hosking, Stephanie Howe, Eric Hu, Renjie Huang, Hemant Jain, Ritika Jain, Nick Jakobi, Madeline Jenkins, JJ Jordan, Dhruti Joshi, Jason Jung, Trushant Kalyanpur, Siddhartha Rao Kamalakara, Julia Kedrzycki, Gokce Keskin, Edward Kim, Joon Kim, Wei-Yin Ko, Tom Kocmi, Michael Kozakov, Wojciech Kryściński, Arnav Kumar Jain, Komal Kumar Teru, Sander Land, Michael Lasby, Olivia Lasche, Justin Lee, Patrick Lewis, Jeffrey Li, Jonathan Li, Hangyu Lin, Acyr Locatelli, Kevin Luong, Raymond Ma, Lukáš Mach, Marina Machado, Joanne Magbitang, Brenda Malacara Lopez, Aryan Mann, Kelly Marchisio, Olivia Markham, Alexandre Matton, Alex McKinney, Dominic McLoughlin, Jozef Mokry, Adrien Morisot, Autumn Moulder, Harry Moynehan, Maximilian Mozes, Vivek Muppalla, Lidiya Murakhovska, Hemangani Nagarajan, Alekhya Nandula, Hisham Nasir, Shauna Nehra, Josh Netto-Rosen, Daniel Ohashi, James Owers-Bardsley, Jason Ozuzu, Dennis Padilla, Gloria Park, Sam Passaglia, Jeremy Pekmez, Laura Penstone, Aleksandra Piktus, Case Ploeg, Andrew Poulton, Youran Qi, Shubha Raghvendra, Miguel Ramos, Ekagra Ranjan, Pierre Richemond, Cécile Robert-Michon, Aurélien Rodriguez, Sudip Roy, Sebastian Ruder, Laura Ruis, Louise Rust, Anubhav Sachan, Alejandro Salamanca, Kailash Karthik Saravanakumar, Isha Satyakam, Alice Schoenauer Sebag, Priyanka Sen, Sholeh Sepehri, Preethi Seshadri, Ye Shen, Tom Sherborne, Sylvie Shang Shi, Sanal Shivaprasad, Vladyslav Shmyhlo, Anirudh Shrinivason, Inna Shteinbuk, Amir Shukayev, Mathieu Simard, Ella Snyder, Ava Spataru, Victoria Spooner, Trisha Starostina, Florian Strub, Yixuan Su, Jimin Sun, Dwarak Talupuru, Eugene Tarassov, Elena Tommasone, Jennifer Tracey, Billy Trend, Evren Tumer, Ahmet Üstün, Bharat Venkitesh, David Venuto, Pat Verga, Maxime Voisin, Alex Wang, Donglu Wang, Shijian Wang, Edmond Wen, Naomi White, Jesse Willman, Marysia Winkels, Chen Xia, Jessica Xie, Minjie Xu, Bowen Yang, Tan Yi-Chern, Ivan Zhang, Zhenyu Zhao, and Zhoujie Zhao

2025

Bib Paper Model

@misc{cohere2025commandaenterprisereadylarge,
  title = {Command A: An Enterprise-Ready Large Language Model},
  author = {Cohere, Team and Aakanksha and Ahmadian, Arash and Ahmed, Marwan and Alammar, Jay and Alizadeh, Milad and Alnumay, Yazeed and Althammer, Sophia and Arkhangorodsky, Arkady and Aryabumi, Viraat and Aumiller, Dennis and Avalos, Raphaël and Aviv, Zahara and Bae, Sammie and Baji, Saurabh and Barbet, Alexandre and Bartolo, Max and Bebensee, Björn and Beladia, Neeral and Beller-Morales, Walter and Bérard, Alexandre and Berneshawi, Andrew and Bialas, Anna and Blunsom, Phil and Bobkin, Matt and Bongale, Adi and Braun, Sam and Brunet, Maxime and Cahyawijaya, Samuel and Cairuz, David and Campos, Jon Ander and Cao, Cassie and Cao, Kris and Castagné, Roman and Cendrero, Julián and Currie, Leila Chan and Chandak, Yash and Chang, Diane and Chatziveroglou, Giannis and Chen, Hongyu and Cheng, Claire and Chevalier, Alexis and Chiu, Justin T. and Cho, Eugene and Choi, Eugene and Choi, Eujeong and Chung, Tim and Cirik, Volkan and Cismaru, Ana and Clavier, Pierre and Conklin, Henry and Crawhall-Stein, Lucas and Crouse, Devon and Cruz-Salinas, Andres Felipe and Cyrus, Ben and D'souza, Daniel and Dalla-Torre, Hugo and Dang, John and Darling, William and Domingues, Omar Darwiche and Dash, Saurabh and Debugne, Antoine and Dehaze, Théo and Desai, Shaan and Devassy, Joan and Dholakia, Rishit and Duffy, Kyle and Edalati, Ali and Eldeib, Ace and Elkady, Abdullah and Elsharkawy, Sarah and Ergün, Irem and Ermis, Beyza and Fadaee, Marzieh and Fan, Boyu and Fayoux, Lucas and Flet-Berliac, Yannis and Frosst, Nick and Gallé, Matthias and Galuba, Wojciech and Garg, Utsav and Geist, Matthieu and Azar, Mohammad Gheshlaghi and Gilsenan-McMahon, Ellen and Goldfarb-Tarrant, Seraphina and Goldsack, Tomas and Gomez, Aidan and Gonzaga, Victor Machado and Govindarajan, Nithya and Govindassamy, Manoj and Grinsztajn, Nathan and Gritsch, Nikolas and Gu, Patrick and Guo, Shangmin and Haefeli, Kilian and Hajjar, Rod and Hawes, Tim and He, Jingyi and Hofstätter, Sebastian and Hong, Sungjin and Hooker, Sara and Hosking, Tom and Howe, Stephanie and Hu, Eric and Huang, Renjie and Jain, Hemant and Jain, Ritika and Jakobi, Nick and Jenkins, Madeline and Jordan, JJ and Joshi, Dhruti and Jung, Jason and Kalyanpur, Trushant and Kamalakara, Siddhartha Rao and Kedrzycki, Julia and Keskin, Gokce and Kim, Edward and Kim, Joon and Ko, Wei-Yin and Kocmi, Tom and Kozakov, Michael and Kryściński, Wojciech and Jain, Arnav Kumar and Teru, Komal Kumar and Land, Sander and Lasby, Michael and Lasche, Olivia and Lee, Justin and Lewis, Patrick and Li, Jeffrey and Li, Jonathan and Lin, Hangyu and Locatelli, Acyr and Luong, Kevin and Ma, Raymond and Mach, Lukáš and Machado, Marina and Magbitang, Joanne and Lopez, Brenda Malacara and Mann, Aryan and Marchisio, Kelly and Markham, Olivia and Matton, Alexandre and McKinney, Alex and McLoughlin, Dominic and Mokry, Jozef and Morisot, Adrien and Moulder, Autumn and Moynehan, Harry and Mozes, Maximilian and Muppalla, Vivek and Murakhovska, Lidiya and Nagarajan, Hemangani and Nandula, Alekhya and Nasir, Hisham and Nehra, Shauna and Netto-Rosen, Josh and Ohashi, Daniel and Owers-Bardsley, James and Ozuzu, Jason and Padilla, Dennis and Park, Gloria and Passaglia, Sam and Pekmez, Jeremy and Penstone, Laura and Piktus, Aleksandra and Ploeg, Case and Poulton, Andrew and Qi, Youran and Raghvendra, Shubha and Ramos, Miguel and Ranjan, Ekagra and Richemond, Pierre and Robert-Michon, Cécile and Rodriguez, Aurélien and Roy, Sudip and Ruder, Sebastian and Ruis, Laura and Rust, Louise and Sachan, Anubhav and Salamanca, Alejandro and Saravanakumar, Kailash Karthik and Satyakam, Isha and Sebag, Alice Schoenauer and Sen, Priyanka and Sepehri, Sholeh and Seshadri, Preethi and Shen, Ye and Sherborne, Tom and Shi, Sylvie Shang and Shivaprasad, Sanal and Shmyhlo, Vladyslav and Shrinivason, Anirudh and Shteinbuk, Inna and Shukayev, Amir and Simard, Mathieu and Snyder, Ella and Spataru, Ava and Spooner, Victoria and Starostina, Trisha and Strub, Florian and Su, Yixuan and Sun, Jimin and Talupuru, Dwarak and Tarassov, Eugene and Tommasone, Elena and Tracey, Jennifer and Trend, Billy and Tumer, Evren and Üstün, Ahmet and Venkitesh, Bharat and Venuto, David and Verga, Pat and Voisin, Maxime and Wang, Alex and Wang, Donglu and Wang, Shijian and Wen, Edmond and White, Naomi and Willman, Jesse and Winkels, Marysia and Xia, Chen and Xie, Jessica and Xu, Minjie and Yang, Bowen and Yi-Chern, Tan and Zhang, Ivan and Zhao, Zhenyu and Zhao, Zhoujie},
  year = {2025},
  eprint = {2504.00698},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

From Tools to Teammates: Evaluating LLMs in Multi-Session Coding Interactions

Nathanaël Carraz Rakotonirina, Mohammed Hamdy, Jon Ander Campos, Lucas Weber, Alberto Testoni, Marzieh Fadaee, Sandro Pezzelle, and Marco Del Tredici

2025

Bib Paper Data Code

@misc{rakotonirina2025toolsteammatesevaluatingllms,
  title = {From Tools to Teammates: Evaluating LLMs in Multi-Session Coding Interactions},
  author = {Rakotonirina, Nathanaël Carraz and Hamdy, Mohammed and Campos, Jon Ander and Weber, Lucas and Testoni, Alberto and Fadaee, Marzieh and Pezzelle, Sandro and Tredici, Marco Del},
  year = {2025},
  eprint = {2502.13791},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

ArXiv

Towards Best Practices for Open Datasets for LLM Training

Stefan Baack, Stella Biderman, Kasia Odrozek, Aviya Skowron, Ayah Bdeir, Jillian Bommarito, Jennifer Ding, Maximilian Gahntz, Paul Keller, Pierre-Carl Langlais, Greg Lindahl, Sebastian Majstorovic, Nik Marda, Guilherme Penedo, Maarten Van Segbroeck, Jennifer Wang, Leandro Werra, Mitchell Baker, Julie Belião, Kasia Chmielinski, Marzieh Fadaee, Lisa Gutermuth, Hynek Kydlíček, Greg Leppert, EM Lewis-Jong, Solana Larsen, Shayne Longpre, Angela Oduor Lungati, Cullen Miller, Victor Miller, Max Ryabinin, Kathleen Siminyu, Andrew Strait, Mark Surman, Anna Tumadóttir, Maurice Weber, Rebecca Weiss, Lee White, and Thomas Wolf

2025

  title = {Towards Best Practices for Open Datasets for LLM Training},
  author = {Baack, Stefan and Biderman, Stella and Odrozek, Kasia and Skowron, Aviya and Bdeir, Ayah and Bommarito, Jillian and Ding, Jennifer and Gahntz, Maximilian and Keller, Paul and Langlais, Pierre-Carl and Lindahl, Greg and Majstorovic, Sebastian and Marda, Nik and Penedo, Guilherme and Segbroeck, Maarten Van and Wang, Jennifer and von Werra, Leandro and Baker, Mitchell and Belião, Julie and Chmielinski, Kasia and Fadaee, Marzieh and Gutermuth, Lisa and Kydlíček, Hynek and Leppert, Greg and Lewis-Jong, EM and Larsen, Solana and Longpre, Shayne and Lungati, Angela Oduor and Miller, Cullen and Miller, Victor and Ryabinin, Max and Siminyu, Kathleen and Strait, Andrew and Surman, Mark and Tumadóttir, Anna and Weber, Maurice and Weiss, Rebecca and White, Lee and Wolf, Thomas},
  year = {2025},
  eprint = {2501.08365},
  archiveprefix = {arXiv},
  primaryclass = {cs.CY},
}

2024

ArXiv

Aya Expanse: Combining Research Breakthroughs for a New Multilingual Frontier

John Dang, Shivalika Singh, Daniel D’souza, Arash Ahmadian, Alejandro Salamanca, Madeline Smith, Aidan Peppin, Sungjin Hong, Manoj Govindassamy, Terrence Zhao, Sandra Kublik, Meor Amer, Viraat Aryabumi, Jon Ander Campos, Yi-Chern Tan, Tom Kocmi, Florian Strub, Nathan Grinsztajn, Yannis Flet-Berliac, Acyr Locatelli, Hangyu Lin, Dwarak Talupuru, Bharat Venkitesh, David Cairuz, Bowen Yang, Tim Chung, Wei-Yin Ko, Sylvie Shang Shi, Amir Shukayev, Sammie Bae, Aleksandra Piktus, Roman Castagné, Felipe Cruz-Salinas, Eddie Kim, Lucas Crawhall-Stein, Adrien Morisot, Sudip Roy, Phil Blunsom, Ivan Zhang, Aidan Gomez, Nick Frosst, Marzieh Fadaee, Beyza Ermis, Ahmet Üstün, and Sara Hooker

2024

Bib Paper Supp Model

@misc{dang2024ayaexpansecombiningresearch,
  title = {Aya Expanse: Combining Research Breakthroughs for a New Multilingual Frontier},
  author = {Dang, John and Singh, Shivalika and D'souza, Daniel and Ahmadian, Arash and Salamanca, Alejandro and Smith, Madeline and Peppin, Aidan and Hong, Sungjin and Govindassamy, Manoj and Zhao, Terrence and Kublik, Sandra and Amer, Meor and Aryabumi, Viraat and Campos, Jon Ander and Tan, Yi-Chern and Kocmi, Tom and Strub, Florian and Grinsztajn, Nathan and Flet-Berliac, Yannis and Locatelli, Acyr and Lin, Hangyu and Talupuru, Dwarak and Venkitesh, Bharat and Cairuz, David and Yang, Bowen and Chung, Tim and Ko, Wei-Yin and Shi, Sylvie Shang and Shukayev, Amir and Bae, Sammie and Piktus, Aleksandra and Castagné, Roman and Cruz-Salinas, Felipe and Kim, Eddie and Crawhall-Stein, Lucas and Morisot, Adrien and Roy, Sudip and Blunsom, Phil and Zhang, Ivan and Gomez, Aidan and Frosst, Nick and Fadaee, Marzieh and Ermis, Beyza and Üstün, Ahmet and Hooker, Sara},
  year = {2024},
  eprint = {2412.04261},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation

Shivalika Singh, Angelika Romanou, Clémentine Fourrier, David I. Adelani, Jian Gang Ngui, Daniel Vila-Suero, Peerat Limkonchotiwat, Kelly Marchisio, Wei Qi Leong, Yosephine Susanto, Raymond Ng, Shayne Longpre, Wei-Yin Ko, Madeline Smith, Antoine Bosselut, Alice Oh, Andre F. T. Martins, Leshem Choshen, Daphne Ippolito, Enzo Ferrante, Marzieh Fadaee, Beyza Ermis, and Sara Hooker

2024

@misc{singh2024globalmmluunderstandingaddressing,
  title = {Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation},
  author = {Singh, Shivalika and Romanou, Angelika and Fourrier, Clémentine and Adelani, David I. and Ngui, Jian Gang and Vila-Suero, Daniel and Limkonchotiwat, Peerat and Marchisio, Kelly and Leong, Wei Qi and Susanto, Yosephine and Ng, Raymond and Longpre, Shayne and Ko, Wei-Yin and Smith, Madeline and Bosselut, Antoine and Oh, Alice and Martins, Andre F. T. and Choshen, Leshem and Ippolito, Daphne and Ferrante, Enzo and Fadaee, Marzieh and Ermis, Beyza and Hooker, Sara},
  year = {2024},
  eprint = {2412.03304},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

ICLR

INCLUDE: Evaluating Multilingual Language Understanding with Regional Knowledge

Angelika Romanou, Negar Foroutan, Anna Sotnikova, Zeming Chen, Sree Harsha Nelaturu, Shivalika Singh, Rishabh Maheshwary, Micol Altomare, Mohamed A. Haggag, Snegha A, Alfonso Amayuelas, Azril Hafizi Amirudin, Viraat Aryabumi, Danylo Boiko, Michael Chang, Jenny Chim, Gal Cohen, Aditya Kumar Dalmia, Abraham Diress, Sharad Duwal, Daniil Dzenhaliou, Daniel Fernando Erazo Florez, Fabian Farestam, Joseph Marvin Imperial, Shayekh Bin Islam, Perttu Isotalo, Maral Jabbarishiviari, Börje F. Karlsson, Eldar Khalilov, Christopher Klamm, Fajri Koto, Dominik Krzemiński, Gabriel Adriano Melo, Syrielle Montariol, Yiyang Nan, Joel Niklaus, Jekaterina Novikova, Johan Samir Obando Ceron, Debjit Paul, Esther Ploeger, Jebish Purbey, Swati Rajwal, Selvan Sunitha Ravi, Sara Rydell, Roshan Santhosh, Drishti Sharma, Marjana Prifti Skenduli, Arshia Soltani Moakhar, Bardia Soltani Moakhar, Ran Tamir, Ayush Kumar Tarun, Azmine Toushik Wasi, Thenuka Ovin Weerasinghe, Serhan Yilmaz, Mike Zhang, Imanol Schlag, Marzieh Fadaee, Sara Hooker, and Antoine Bosselut

2024

Bib Paper Data Poster

@misc{romanou2024includeevaluatingmultilinguallanguage,
  title = {INCLUDE: Evaluating Multilingual Language Understanding with Regional Knowledge},
  author = {Romanou, Angelika and Foroutan, Negar and Sotnikova, Anna and Chen, Zeming and Nelaturu, Sree Harsha and Singh, Shivalika and Maheshwary, Rishabh and Altomare, Micol and Haggag, Mohamed A. and A, Snegha and Amayuelas, Alfonso and Amirudin, Azril Hafizi and Aryabumi, Viraat and Boiko, Danylo and Chang, Michael and Chim, Jenny and Cohen, Gal and Dalmia, Aditya Kumar and Diress, Abraham and Duwal, Sharad and Dzenhaliou, Daniil and Florez, Daniel Fernando Erazo and Farestam, Fabian and Imperial, Joseph Marvin and Islam, Shayekh Bin and Isotalo, Perttu and Jabbarishiviari, Maral and Karlsson, Börje F. and Khalilov, Eldar and Klamm, Christopher and Koto, Fajri and Krzemiński, Dominik and de Melo, Gabriel Adriano and Montariol, Syrielle and Nan, Yiyang and Niklaus, Joel and Novikova, Jekaterina and Ceron, Johan Samir Obando and Paul, Debjit and Ploeger, Esther and Purbey, Jebish and Rajwal, Swati and Ravi, Selvan Sunitha and Rydell, Sara and Santhosh, Roshan and Sharma, Drishti and Skenduli, Marjana Prifti and Moakhar, Arshia Soltani and Moakhar, Bardia Soltani and Tamir, Ran and Tarun, Ayush Kumar and Wasi, Azmine Toushik and Weerasinghe, Thenuka Ovin and Yilmaz, Serhan and Zhang, Mike and Schlag, Imanol and Fadaee, Marzieh and Hooker, Sara and Bosselut, Antoine},
  year = {2024},
  eprint = {2411.19799},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

M-RewardBench: Evaluating Reward Models in Multilingual Settings

Srishti Gureja, Lester James V. Miranda, Shayekh Bin Islam, Rishabh Maheshwary, Drishti Sharma, Gusti Winata, Nathan Lambert, Sebastian Ruder, Sara Hooker, and Marzieh Fadaee

2024

Bib Paper Data Code

  title = {M-RewardBench: Evaluating Reward Models in Multilingual Settings},
  author = {Gureja, Srishti and Miranda, Lester James V. and Islam, Shayekh Bin and Maheshwary, Rishabh and Sharma, Drishti and Winata, Gusti and Lambert, Nathan and Ruder, Sebastian and Hooker, Sara and Fadaee, Marzieh},
  year = {2024},
  eprint = {2410.15522},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

Neurips Wokshop

Mix Data or Merge Models? Optimizing for Diverse Multi-Task Learning

Aakanksha, Arash Ahmadian, Seraphina Goldfarb-Tarrant, Beyza Ermis, Marzieh Fadaee, and Sara Hooker

2024

  title = {Mix Data or Merge Models? Optimizing for Diverse Multi-Task Learning},
  author = {Aakanksha and Ahmadian, Arash and Goldfarb-Tarrant, Seraphina and Ermis, Beyza and Fadaee, Marzieh and Hooker, Sara},
  year = {2024},
  eprint = {2410.10801},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

ArXiv

Diversify and Conquer: Diversity-Centric Data Selection with Iterative Refinement

Simon Yu, Liangyu Chen, Sara Ahmadian, and Marzieh Fadaee

2024

Bib Paper Code

  title = {Diversify and Conquer: Diversity-Centric Data Selection with Iterative Refinement},
  author = {Yu, Simon and Chen, Liangyu and Ahmadian, Sara and Fadaee, Marzieh},
  year = {2024},
  eprint = {2409.11378},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

ICLR

To Code, or Not To Code? Exploring Impact of Code in Pre-training

Viraat Aryabumi, Yixuan Su, Raymond Ma, Adrien Morisot, Ivan Zhang, Acyr Locatelli, Marzieh Fadaee, Ahmet Üstün, and Sara Hooker

2024

Bib Paper Poster

  title = {To Code, or Not To Code? Exploring Impact of Code in Pre-training},
  author = {Aryabumi, Viraat and Su, Yixuan and Ma, Raymond and Morisot, Adrien and Zhang, Ivan and Locatelli, Acyr and Fadaee, Marzieh and Üstün, Ahmet and Hooker, Sara},
  year = {2024},
  eprint = {2408.10914},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

EMNLP

LLM See, LLM Do: Guiding Data Generation to Target Non-Differentiable Objectives

Luísa Shimabucoro, Sebastian Ruder, Julia Kreutzer, Marzieh Fadaee, and Sara Hooker

2024

@misc{shimabucoro2024llmseellmdo,
  title = {LLM See, LLM Do: Guiding Data Generation to Target Non-Differentiable Objectives},
  author = {Shimabucoro, Luísa and Ruder, Sebastian and Kreutzer, Julia and Fadaee, Marzieh and Hooker, Sara},
  year = {2024},
  eprint = {2407.01490},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

EMNLP

The Multilingual Alignment Prism: Aligning Global and Local Preferences to Reduce Harm

Aakanksha, Arash Ahmadian, Beyza Ermis, Seraphina Goldfarb-Tarrant, Julia Kreutzer, Marzieh Fadaee, and Sara Hooker

2024

@misc{aakanksha2024multilingualalignmentprismaligning,
  title = {The Multilingual Alignment Prism: Aligning Global and Local Preferences to Reduce Harm},
  author = {Aakanksha and Ahmadian, Arash and Ermis, Beyza and Goldfarb-Tarrant, Seraphina and Kreutzer, Julia and Fadaee, Marzieh and Hooker, Sara},
  year = {2024},
  eprint = {2406.18682},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

ArXiv

Aya 23: Open Weight Releases to Further Multilingual Progress

Viraat Aryabumi, John Dang, Dwarak Talupuru, Saurabh Dash, David Cairuz, Hangyu Lin, Bharat Venkitesh, Madeline Smith, Jon Ander Campos, Yi Chern Tan, Kelly Marchisio, Max Bartolo, Sebastian Ruder, Acyr Locatelli, Julia Kreutzer, Nick Frosst, Aidan Gomez, Phil Blunsom, Marzieh Fadaee, Ahmet Üstün, and Sara Hooker

2024

Bib Paper Model

@misc{aryabumi2024aya23openweight,
  title = {Aya 23: Open Weight Releases to Further Multilingual Progress},
  author = {Aryabumi, Viraat and Dang, John and Talupuru, Dwarak and Dash, Saurabh and Cairuz, David and Lin, Hangyu and Venkitesh, Bharat and Smith, Madeline and Campos, Jon Ander and Tan, Yi Chern and Marchisio, Kelly and Bartolo, Max and Ruder, Sebastian and Locatelli, Acyr and Kreutzer, Julia and Frosst, Nick and Gomez, Aidan and Blunsom, Phil and Fadaee, Marzieh and Üstün, Ahmet and Hooker, Sara},
  year = {2024},
  eprint = {2405.15032},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

Back to Basics: Revisiting REINFORCE Style Optimization for Learning from Human Feedback in LLMs

Arash Ahmadian, Chris Cremer, Matthias Gallé, Marzieh Fadaee, Julia Kreutzer, Ahmet Üstün, and Sara Hooker

2024

@misc{ahmadian2024basics,
  title = {Back to Basics: Revisiting REINFORCE Style Optimization for Learning from Human Feedback in LLMs},
  author = {Ahmadian, Arash and Cremer, Chris and Gallé, Matthias and Fadaee, Marzieh and Kreutzer, Julia and Üstün, Ahmet and Hooker, Sara},
  year = {2024},
  eprint = {2402.14740},
  archiveprefix = {arXiv},
  primaryclass = {cs.LG},
}

Aya Model: An Instruction Finetuned Open-Access Multilingual Language Model

Ahmet Üstün, Viraat Aryabumi, Zheng-Xin Yong, Wei-Yin Ko, Daniel D’souza, Gbemileke Onilude, Neel Bhandari, Shivalika Singh, Hui-Lee Ooi, Amr Kayid, Freddie Vargus, Phil Blunsom, Shayne Longpre, Niklas Muennighoff, Marzieh Fadaee, Julia Kreutzer, and Sara Hooker

2024

Bib Paper Model

@misc{üstün2024aya,
  title = {Aya Model: An Instruction Finetuned Open-Access Multilingual Language Model},
  author = {Üstün, Ahmet and Aryabumi, Viraat and Yong, Zheng-Xin and Ko, Wei-Yin and D'souza, Daniel and Onilude, Gbemileke and Bhandari, Neel and Singh, Shivalika and Ooi, Hui-Lee and Kayid, Amr and Vargus, Freddie and Blunsom, Phil and Longpre, Shayne and Muennighoff, Niklas and Fadaee, Marzieh and Kreutzer, Julia and Hooker, Sara},
  year = {2024},
  eprint = {2402.07827},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
  award = {Best paper award},
}

Aya Dataset: An Open-Access Collection for Multilingual Instruction Tuning

Shivalika Singh, Freddie Vargus, Daniel Dsouza, Börje F. Karlsson, Abinaya Mahendiran, Wei-Yin Ko, Herumb Shandilya, Jay Patel, Deividas Mataciunas, Laura OMahony, Mike Zhang, Ramith Hettiarachchi, Joseph Wilson, Marina Machado, Luisa Souza Moura, Dominik Krzemiński, Hakimeh Fadaei, Irem Ergün, Ifeoma Okoh, Aisha Alaagib, Oshan Mudannayake, Zaid Alyafeai, Vu Minh Chien, Sebastian Ruder, Surya Guthikonda, Emad A. Alghamdi, Sebastian Gehrmann, Niklas Muennighoff, Max Bartolo, Julia Kreutzer, Ahmet Üstün, Marzieh Fadaee, and Sara Hooker

2024

@misc{singh2024aya,
  title = {Aya Dataset: An Open-Access Collection for Multilingual Instruction Tuning},
  author = {Singh, Shivalika and Vargus, Freddie and Dsouza, Daniel and Karlsson, Börje F. and Mahendiran, Abinaya and Ko, Wei-Yin and Shandilya, Herumb and Patel, Jay and Mataciunas, Deividas and OMahony, Laura and Zhang, Mike and Hettiarachchi, Ramith and Wilson, Joseph and Machado, Marina and Souza Moura, Luisa and Krzemiński, Dominik and Fadaei, Hakimeh and Ergün, Irem and Okoh, Ifeoma and Alaagib, Aisha and Mudannayake, Oshan and Alyafeai, Zaid and Chien, Vu Minh and Ruder, Sebastian and Guthikonda, Surya and Alghamdi, Emad A. and Gehrmann, Sebastian and Muennighoff, Niklas and Bartolo, Max and Kreutzer, Julia and Üstün, Ahmet and Fadaee, Marzieh and Hooker, Sara},
  year = {2024},
  eprint = {2402.06619},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

2023

Neurips

Elo Uncovered: Robustness and Best Practices in Language Model Evaluation

Meriem Boubdir, Edward Kim, Beyza Ermis, Sara Hooker, and Marzieh Fadaee

2023

Bib Paper Poster

@misc{boubdir2023elo,
  title = {Elo Uncovered: Robustness and Best Practices in Language Model Evaluation},
  author = {Boubdir, Meriem and Kim, Edward and Ermis, Beyza and Hooker, Sara and Fadaee, Marzieh},
  year = {2023},
  eprint = {2311.17295},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
}

ArXiv

Which Prompts Make The Difference? Data Prioritization For Efficient Human LLM Evaluation

Meriem Boubdir, Edward Kim, Beyza Ermis, Marzieh Fadaee, and Sara Hooker

2023