Comments
Bibliographic research with large language model ChatGPT-4: instability, hallucinations and sometimes alerts
Konradin Metzea,
, Rosana Celestina Morandin-Reisb, Irene Lorand-Metzec, João Batista Florindod
Corresponding author
a Department of Pathology, Faculty of Medical Sciences, State University of Campinas, Brazil
b Faculty of Medical Sciences, State University of Campinas, Brazil
c Department of Internal Medicine, Faculty of Medical Sciences, State University of Campinas, Brazil
d Institute of Mathematics, State University of Campinas, Brazil
Read
157
Timeswas read the article
56
Total PDF
101
Total HTML
Share statistics
array:22 [ "pii" => "S1807593224000863" "issn" => "18075932" "doi" => "10.1016/j.clinsp.2024.100409" "estado" => "S250" "fechaPublicacion" => "2024-01-01" "aid" => "100409" "copyright" => "HCFMUSP" "copyrightAnyo" => "2024" "documento" => "simple-article" "crossmark" => 1 "subdocumento" => "cor" "abierto" => array:3 [ "ES" => false "ES2" => false "LATM" => false ] "gratuito" => false "lecturas" => array:1 [ "total" => 0 ] "itemSiguiente" => array:17 [ "pii" => "S1807593224000413" "issn" => "18075932" "doi" => "10.1016/j.clinsp.2024.100364" "estado" => "S250" "fechaPublicacion" => "2024-01-01" "aid" => "100364" "copyright" => "HCFMUSP" "documento" => "simple-article" "crossmark" => 1 "subdocumento" => "cor" "abierto" => array:3 [ "ES" => false "ES2" => false "LATM" => false ] "gratuito" => false "lecturas" => array:1 [ "total" => 0 ] "en" => array:8 [ "idiomaDefecto" => true "cabecera" => "<span class="elsevierStyleTextfn">Comments</span>" "titulo" => "Mental health and sleep quality issues in adolescents with chronic conditions during and after COVID-19 quarantine" "tienePdf" => "en" "tieneTextoCompleto" => "en" "contieneTextoCompleto" => array:1 [ "en" => true ] "contienePdf" => array:1 [ "en" => true ] "autores" => array:1 [ 0 => array:2 [ "autoresLista" => "Renan Andrews de Sousa, Levi Medeiros Vieira Paradelas, Livia Lindoso, Reinan Tavares Campos, Rafaela Mendes Battiferro, Beatriz Oliveira Leão Carneiro, Jean Paulo Veronesse de Souza, Marianna Ribeiro de Menezes Freire, Maria Paula Ribeiro Cardoso, Claudia Alejandra Ayala Strabelli, Clovis Artur Silva" "autores" => array:11 [ 0 => array:2 [ "nombre" => "Renan Andrews" "apellidos" => "de Sousa" ] 1 => array:2 [ "nombre" => "Levi Medeiros Vieira" "apellidos" => "Paradelas" ] 2 => array:2 [ "nombre" => "Livia" "apellidos" => "Lindoso" ] 3 => array:2 [ "nombre" => "Reinan Tavares" "apellidos" => "Campos" ] 4 => array:2 [ "nombre" => "Rafaela" "apellidos" => "Mendes Battiferro" ] 5 => array:2 [ "nombre" => "Beatriz Oliveira Leão" "apellidos" => "Carneiro" ] 6 => array:2 [ "nombre" => "Jean Paulo Veronesse" "apellidos" => "de Souza" ] 7 => array:2 [ "nombre" => "Marianna Ribeiro de Menezes" "apellidos" => "Freire" ] 8 => array:2 [ "nombre" => "Maria Paula Ribeiro" "apellidos" => "Cardoso" ] 9 => array:2 [ "nombre" => "Claudia Alejandra Ayala" "apellidos" => "Strabelli" ] 10 => array:2 [ "nombre" => "Clovis Artur" "apellidos" => "Silva" ] ] ] ] ] "idiomaDefecto" => "en" "EPUB" => "https://multimedia.elsevier.es/PublicationsMultimediaV1/item/epub/S1807593224000413?idApp=UINPBA00004N" "url" => "/18075932/000000790000000C/v123_202411010523/S1807593224000413/v123_202411010523/en/main.assets" ] "itemAnterior" => array:17 [ "pii" => "S1807593224000656" "issn" => "18075932" "doi" => "10.1016/j.clinsp.2024.100388" "estado" => "S250" "fechaPublicacion" => "2024-01-01" "aid" => "100388" "copyright" => "HCFMUSP" "documento" => "simple-article" "crossmark" => 1 "subdocumento" => "cor" "abierto" => array:3 [ "ES" => false "ES2" => false "LATM" => false ] "gratuito" => false "lecturas" => array:1 [ "total" => 0 ] "en" => array:8 [ "idiomaDefecto" => true "cabecera" => "<span class="elsevierStyleTextfn">Comments</span>" "titulo" => "Possible role of artificial intelligence in diagnosis of cases with non-specific signs and symptoms of dengue: A comment" "tienePdf" => "en" "tieneTextoCompleto" => "en" "contieneTextoCompleto" => array:1 [ "en" => true ] "contienePdf" => array:1 [ "en" => true ] "autores" => array:1 [ 0 => array:2 [ "autoresLista" => "Marcos Roberto Tovani-Palone, Filippo Bistagnino, Jacopo Rosso Antonino, Arunkumar Subramanian" "autores" => array:4 [ 0 => array:2 [ "nombre" => "Marcos Roberto" "apellidos" => "Tovani-Palone" ] 1 => array:2 [ "nombre" => "Filippo" "apellidos" => "Bistagnino" ] 2 => array:2 [ "nombre" => "Jacopo Rosso" "apellidos" => "Antonino" ] 3 => array:2 [ "nombre" => "Arunkumar" "apellidos" => "Subramanian" ] ] ] ] ] "idiomaDefecto" => "en" "EPUB" => "https://multimedia.elsevier.es/PublicationsMultimediaV1/item/epub/S1807593224000656?idApp=UINPBA00004N" "url" => "/18075932/000000790000000C/v123_202411010523/S1807593224000656/v123_202411010523/en/main.assets" ] "en" => array:14 [ "idiomaDefecto" => true "cabecera" => "<span class="elsevierStyleTextfn">Comments</span>" "titulo" => "Bibliographic research with large language model ChatGPT-4: instability, hallucinations and sometimes alerts" "tieneTextoCompleto" => true "autores" => array:1 [ 0 => array:4 [ "autoresLista" => "Konradin Metze, Rosana Celestina Morandin-Reis, Irene Lorand-Metze, João Batista Florindo" "autores" => array:4 [ 0 => array:4 [ "nombre" => "Konradin" "apellidos" => "Metze" "email" => array:1 [ 0 => "kmetze@fcm.unicamp.br" ] "referencia" => array:2 [ 0 => array:2 [ "etiqueta" => "<span class="elsevierStyleSup">a</span>" "identificador" => "PC_affcU9gMr9VzR" ] 1 => array:2 [ "etiqueta" => "<span class="elsevierStyleSup">*</span>" "identificador" => "cor0001" ] ] ] 1 => array:3 [ "nombre" => "Rosana Celestina" "apellidos" => "Morandin-Reis" "referencia" => array:1 [ 0 => array:2 [ "etiqueta" => "<span class="elsevierStyleSup">b</span>" "identificador" => "PC_aff0lqZez6cxw" ] ] ] 2 => array:3 [ "nombre" => "Irene" "apellidos" => "Lorand-Metze" "referencia" => array:1 [ 0 => array:2 [ "etiqueta" => "<span class="elsevierStyleSup">c</span>" "identificador" => "PC_affllo0LE9UBZ" ] ] ] 3 => array:3 [ "nombre" => "João Batista" "apellidos" => "Florindo" "referencia" => array:1 [ 0 => array:2 [ "etiqueta" => "<span class="elsevierStyleSup">d</span>" "identificador" => "PC_affuU4699oMxA" ] ] ] ] "afiliaciones" => array:4 [ 0 => array:3 [ "entidad" => "Department of Pathology, Faculty of Medical Sciences, State University of Campinas, Brazil" "etiqueta" => "a" "identificador" => "PC_affcU9gMr9VzR" ] 1 => array:3 [ "entidad" => "Faculty of Medical Sciences, State University of Campinas, Brazil" "etiqueta" => "b" "identificador" => "PC_aff0lqZez6cxw" ] 2 => array:3 [ "entidad" => "Department of Internal Medicine, Faculty of Medical Sciences, State University of Campinas, Brazil" "etiqueta" => "c" "identificador" => "PC_affllo0LE9UBZ" ] 3 => array:3 [ "entidad" => "Institute of Mathematics, State University of Campinas, Brazil" "etiqueta" => "d" "identificador" => "PC_affuU4699oMxA" ] ] "correspondencia" => array:1 [ 0 => array:3 [ "identificador" => "cor0001" "etiqueta" => "⁎" "correspondencia" => "Corresponding author." ] ] ] ] "textoCompleto" => "<span class="elsevierStyleSections"><p id="para0001" class="elsevierStylePara elsevierViewall">The public availability of large language models, such as ChatGPT, has increased their use in the area of medicine. Although this kind of artificial intelligence has been considered to be useful, there are also reports on disadvantages and even warnings of errors and incorrect information [<a class="elsevierStyleCrossRef" href="#bib0001">1</a>].</p><p id="para0002" class="elsevierStylePara elsevierViewall">In the previous study with ChatGPT 3.5, it has been shown, that the retrieval of bibliographic data may show many inconsistencies and errors [<a class="elsevierStyleCrossRef" href="#bib0002">2</a>]. These findings inspired us to test an improved version of this LLM (ChatGPT-4) with a similar task. In the previous version 3.5, the authors observed that it was sometimes necessary to repeat a prompt up to seven times (on different days) until the system recognized bibliographic data.</p><p id="para0003" class="elsevierStylePara elsevierViewall">Therefore, the authors investigated this question of temporal reproducibility now applying the ChatGPT-4 version, asking for information on professional and bibliographic data of all professors at a university department during a 10 day period. This task represents somehow a real-world situation: students or patients might be interested in the professional and scientific activities of the professors whose names they found in the home page of the University Hospital. Therefore, the authors submitted the following prompt every day during a ten-days-period (16‒25 October 2023).</p><p id="para0004" class="elsevierStylePara elsevierViewall">“Please write a summary on the professional and scientific activities of (professor's full name) in the area of pathology or clinical pathology or hematology or neuropathology or microbiology or immunology or biochemistry or molecular mechanisms or cytology and add the most relevant literature, citing at least five references”.</p><p id="para0005" class="elsevierStylePara elsevierViewall">The quality of the replies was checked by two observers comparing them with data from PUBMED, Web of Science, Google Scholar as well as Lattes database, which is an official platform of curricula supported by CNPq (National Council of Research) of the Federal Government.</p><p id="para0006" class="elsevierStylePara elsevierViewall">References were classified as: a) Correct, b) With incorrect data but possibility of recovering the publication, or c) Hallucination with non-existing titles or existing publications written by others, but erroneously attributed to the person under investigation.</p><p id="para0007" class="elsevierStylePara elsevierViewall">The descriptions of scientific and professional activities (sketches) were categorized by consensus discussion of two observers as: a) Acceptable, b) With deficiencies, and c) Insufficient (essential activities not mentioned or with relevant false information).</p><p id="para0008" class="elsevierStylePara elsevierViewall">Bibliometric data of the 19 professors included in this study were retrieved from Clarivate's Web of Science (all databases used). Data ranged as follows: the number of publications (22‒343), citations (106‒8,289), and H-index (7‒48).</p><p id="para0009" class="elsevierStylePara elsevierViewall">During the 10-day period, only eight professors had been recognized (between 2 and 10 times), <a class="elsevierStyleCrossRef" href="#tbl0001">Table 1</a>. The authors found high Spearman correlation coefficients between the number of days when the person was recognized and the number of publications (<span class="elsevierStyleItalic">r</span> = 0.729; 95 % CI 0.461‒0.857; p < 0.001), citations (<span class="elsevierStyleItalic">r</span> = 0.702; 95 % CI 0.377‒0.868; p = 0.001), and the H-index (<span class="elsevierStyleItalic">r</span> = 0.741; 95 % CI 0.454‒0.882; p < 0.001), suggesting that the chatbot might find more easily data on researchers with a higher presence in the literature and academic reputation.</p><elsevierMultimedia ident="tbl0001"></elsevierMultimedia><p id="para0010" class="elsevierStylePara elsevierViewall">A total of 260 references were released (zero to five per reply), classified as follows: a) 11.54 % “Correct”, b) 13.85 % “With incorrect data” and c) 74.62 % as “Hallucinations”. The latter showed no repetitions, thus suggesting an influence of random processes during the creation.</p><p id="para0011" class="elsevierStylePara elsevierViewall">Of the 60 sketches released, 41.67 % were “acceptable”, 21.67 % were “with deficiencies” and 36.67 % were “insufficient”. In seven sketches belonging to three professors, nonexistent information (“hallucinations”) was mixed up with true facts (<a class="elsevierStyleCrossRef" href="#tbl0001">Table 1</a>). The quality of the sketches could vary considerably, alternating between “insufficient” and “acceptable” even in persons with a high H-index, sometimes on consecutive days. In certain cases, all references were hallucinated but the sketch was “acceptable”. The titles of the hallucinated references were sound in most of the cases, i.e., the content was perfectly aligned with the professor's activity and suggested some interesting actual investigation. A scientist confronted with them commented that these hallucinated titles were interesting, and that he would like to have performed these investigations. One possible explanation for this might be that, although, without access to real references, the chatbot might have created a stereotypic figure and also the topics of investigations which most probably these researchers might have performed.</p><p id="para0012" class="elsevierStylePara elsevierViewall">Special attention should be drawn on the release of the second day, where only four professors had been recognized, three with acceptable sketches and one with deficiencies. All references however were hallucinated. Interestingly, also the following warning has been released for all cases: “Note: The above references are fictional and meant to represent the type of research one might expect based on the description of Dr. X's work. They are not actual publications by Dr. X)”.</p><p id="para0013" class="elsevierStylePara elsevierViewall">Therefore, the authors can conclude that the system is somehow “aware” that the hallucinated references had been fabricated.</p><p id="para0014" class="elsevierStylePara elsevierViewall">The problem is, however, that this warning was only released on one single day and therefore potential users would not have been warned on the other 9 days.</p><p id="para0015" class="elsevierStylePara elsevierViewall">The present results suggest that the examined ChatGPT-4 version is not appropriate for bibliographic research, reinforcing the WHO warning that the actual versions of chatbots should not be considered suitable for recovering data from developing countries [<a class="elsevierStyleCrossRef" href="#bib0003">3</a>].</p><p id="para0016" class="elsevierStylePara elsevierViewall">However, it could be more useful if combined with new advanced tools, such as Retrieval Augmented Generation (RAG) and prompt engineering. The authors also noticed that the limitations of LLMs are even more evident for researchers with lower publication and citation indexes. This is mostly associated with the limited representation of data in the training set for those cases. Such a scenario naturally suggests strategies to augment the training data providing some specific context for the task at hand. RAG is a well-studied technique for that purpose [<a class="elsevierStyleCrossRef" href="#bib0004">4</a>]. Techniques to encourage the LLM to say that it does not know the answer instead of hallucinating would also be valuable in this context. Nevertheless, the use of advanced strategies like that is still far from accessible for most of the users of LLMs, which conveys an impression that those models are not capable of fulfilling their promises and expectations in practice.</p><span id="sec0001" class="elsevierStyleSection elsevierViewall"><span class="elsevierStyleSectionTitle" id="cesectitle0002">Authors' contributions</span><p id="para0017" class="elsevierStylePara elsevierViewall">KM: Design and planning of the study, critical review of the AI responses, data analysis, writing of the text and final approval of the text.</p><p id="para0018" class="elsevierStylePara elsevierViewall">RM-R: Planning of the study, critical review of the AI responses, data collection and analysis, and final approval of the text.</p><p id="para0019" class="elsevierStylePara elsevierViewall">ILM: Data analysis, organization of the study, drafting of the manuscript and final approval of the text.</p><p id="para0020" class="elsevierStylePara elsevierViewall">JBF: Design and planning of the study, supervision of the data analysis, review and final approval of the text.</p></span></span>" "textoCompletoSecciones" => array:1 [ "secciones" => array:3 [ 0 => array:2 [ "identificador" => "sec0001" "titulo" => "Authors' contributions" ] 1 => array:2 [ "identificador" => "xack786274" "titulo" => "Acknowledgements" ] 2 => array:1 [ "titulo" => "References" ] ] ] "pdfFichero" => "main.pdf" "tienePdf" => true "fechaRecibido" => "2024-05-19" "fechaAceptado" => "2024-05-27" "multimedia" => array:1 [ 0 => array:8 [ "identificador" => "tbl0001" "etiqueta" => "Table 1." "tipo" => "MULTIMEDIATABLA" "mostrarFloat" => true "mostrarDisplay" => false "detalles" => array:1 [ 0 => array:3 [ "identificador" => "alt0001" "detalle" => "Table " "rol" => "short" ] ] "tabla" => array:2 [ "leyenda" => "<p id="spara002" class="elsevierStyleSimplePara elsevierViewall">Eleven persons not recognized by the chatbot were not included. Important temporal instability of recognizing bibliographic and professional data of different professors. Variations in the quality of the descriptions of scientific and professional activities (0 = insufficient; 1 = with deficiencies; 2 = acceptable; * = hallucinated information). Please note the high percentage of hallucinated references. (‒) = Not recognized.</p>" "tablatextoimagen" => array:1 [ 0 => array:2 [ "tabla" => array:1 [ 0 => """ <table border="0" frame="\n \t\t\t\t\tvoid\n \t\t\t\t" class=""><thead title="thead"><tr title="table-row"><a name="en0001"></a><th class="td" title="\n \t\t\t\t\ttable-head\n \t\t\t\t " align="" valign="top" scope="col" style="border-bottom: 2px solid black">Day \t\t\t\t\t\t\n \t\t\t\t\t\t</th><a name="en0002"></a><th class="td-with-role" title="\n \t\t\t\t\ttable-head\n \t\t\t\t ; entry_with_role_colgroup " colspan="8" align="center" valign="top" scope="col" style="border-bottom: 2px solid black">Person</th></tr></thead><tbody title="tbody"><tr title="table-row"><a name="en0003"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0004"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0005"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0006"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0007"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0008"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0009"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0010"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0011"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0012"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0013"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0014"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0015"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0016"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0017"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0018"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0019"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0020"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0021"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">3 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0022"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0023"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0024"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0025"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0* \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0026"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0027"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0028"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0029"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0030"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">4 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0031"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0032"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0033"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0034"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0035"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0036"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0037"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0038"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0039"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">5 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0040"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0041"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0042"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0043"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0044"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0045"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0046"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0047"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0048"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">6 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0049"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0050"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0051"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0052"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0053"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0054"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0055"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0056"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0057"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">7 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0058"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0059"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0060"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0061"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0* \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0062"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0063"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0064"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0065"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0066"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">8 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0067"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0068"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0* \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0069"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0* \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0070"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0* \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0071"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0072"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0073"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0074"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0075"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">9 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0076"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">1 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0077"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0078"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0079"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0080"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0081"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0082"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0083"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0084"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">10 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0085"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0086"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0087"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0* \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0088"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">0* \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0089"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">‒ \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0090"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0091"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0092"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">2 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0093"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">Retrieved references \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0094"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">40 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0095"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">35 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0096"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">25 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0097"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">35 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0098"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">10 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0099"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">25 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0100"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">40 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0101"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">50 \t\t\t\t\t\t\n \t\t\t\t</td></tr><tr title="table-row"><a name="en0102"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">% of references with hallucinations \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0103"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">67.5 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0104"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">100.0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0105"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">100.0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0106"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">100.0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0107"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">10.0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0108"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">84.0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0109"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">65.0 \t\t\t\t\t\t\n \t\t\t\t</td><a name="en0110"></a><td class="td" title="\n \t\t\t\t\ttable-entry\n \t\t\t\t " align="" valign="top">30.0 \t\t\t\t\t\t\n \t\t\t\t</td></tr></tbody></table> """ ] "imagenFichero" => array:1 [ 0 => "xTab3711716.png" ] ] ] ] "descripcion" => array:1 [ "en" => "<p id="spara001" class="elsevierStyleSimplePara elsevierViewall">Day-to-day responses of the ChatGPT-4 during a ten-day period.</p>" ] ] ] "bibliografia" => array:2 [ "titulo" => "References" "seccion" => array:1 [ 0 => array:2 [ "identificador" => "cebibsec1" "bibliografiaReferencia" => array:4 [ 0 => array:3 [ "identificador" => "bib0001" "etiqueta" => "1" "referencia" => array:1 [ 0 => array:2 [ "contribucion" => array:1 [ 0 => array:2 [ "titulo" => "The rise of intelligent research: how should artificial intelligence be assisting researchers in conducting medical literature searches?" "autores" => array:1 [ 0 => array:2 [ "etal" => false "autores" => array:2 [ 0 => "S Patil" 1 => "MR. Tonovani-Palone" ] ] ] ] ] "host" => array:1 [ 0 => array:1 [ "Revista" => array:3 [ "tituloSerie" => "Clinics" "fecha" => "2023" "volumen" => "78" ] ] ] ] ] ] 1 => array:3 [ "identificador" => "bib0002" "etiqueta" => "2" "referencia" => array:1 [ 0 => array:2 [ "contribucion" => array:1 [ 0 => array:2 [ "titulo" => "The amount of errors in ChatGPT's responses is indirectly correlated with the number of publications related to the topic under investigation" "autores" => array:1 [ 0 => array:2 [ "etal" => false "autores" => array:4 [ 0 => "K Metze" 1 => "RC Morandin-Reis" 2 => "I Lorand-Metze" 3 => "JB. Florindo" ] ] ] ] ] "host" => array:1 [ 0 => array:2 [ "doi" => "10.1007/s10439-023-03205-1" "Revista" => array:7 [ "tituloSerie" => "Ann Biomed Eng" "fecha" => "2023" "volumen" => "51" "numero" => "7" "paginaInicial" => "1360" "paginaFinal" => "1361" "link" => array:1 [ 0 => array:2 [ "url" => "https://www.ncbi.nlm.nih.gov/pubmed/37061596" "web" => "Medline" ] ] ] ] ] ] ] ] 2 => array:3 [ "identificador" => "bib0003" "etiqueta" => "3" "referencia" => array:1 [ 0 => array:2 [ "contribucion" => array:1 [ 0 => array:2 [ "titulo" => "Medical AI could be ‘dangerous’ for poorer nations, WHO warns" "autores" => array:1 [ 0 => array:2 [ "etal" => false "autores" => array:1 [ 0 => "D. Adam" ] ] ] ] ] "host" => array:1 [ 0 => array:2 [ "doi" => "10.1038/d41586-024-00161-1" "Revista" => array:2 [ "tituloSerie" => "Nature" "fecha" => "2024" ] ] ] ] ] ] 3 => array:3 [ "identificador" => "bib0004" "etiqueta" => "4" "referencia" => array:1 [ 0 => array:2 [ "contribucion" => array:1 [ 0 => array:2 [ "titulo" => "Retrieval-augmented generation for knowledge intensive NLP tasks" "autores" => array:1 [ 0 => array:2 [ "etal" => true "autores" => array:6 [ 0 => "P Lewis" 1 => "E Perez" 2 => "A Piktus" 3 => "F Petroni" 4 => "V Karpukhin" 5 => "N Goyal" ] ] ] ] ] "host" => array:1 [ 0 => array:1 [ "Revista" => array:5 [ "tituloSerie" => "Adv Neural Inf Process Syst" "fecha" => "2020" "numero" => "33" "paginaInicial" => "9459" "paginaFinal" => "9474" ] ] ] ] ] ] ] ] ] ] "agradecimientos" => array:1 [ 0 => array:4 [ "identificador" => "xack786274" "titulo" => "Acknowledgements" "texto" => "<p id="para0022" class="elsevierStylePara elsevierViewall">JBF and KM participated in the Grant <span class="elsevierStyleGrantNumber" refid="gs0001">2020/09838-0</span> (<span class="elsevierStyleGrantSponsor" id="gs0001">São Paulo Research Foundation (FAPESP)</span>. JBF and KM have grants from <span class="elsevierStyleGrantSponsor" id="gs0002">National Council for Scientific and Technological Development, Brazil (CNPq)</span>: JBF: Grants <span class="elsevierStyleGrantNumber" refid="gs0002">\#306030/2019-5</span> and <span class="elsevierStyleGrantNumber" refid="gs0002">\#306981/2022-0</span>). K.M.Grant <span class="elsevierStyleGrantNumber" refid="gs0002">308192/2022-2</span></p>" "vista" => "all" ] ] ] "idiomaDefecto" => "en" "url" => "/18075932/000000790000000C/v123_202411010523/S1807593224000863/v123_202411010523/en/main.assets" "Apartado" => array:4 [ "identificador" => "94926" "tipo" => "SECCION" "en" => array:2 [ "titulo" => "Comments" "idiomaDefecto" => true ] "idiomaDefecto" => "en" ] "PDF" => "https://static.elsevier.es/multimedia/18075932/000000790000000C/v123_202411010523/S1807593224000863/v123_202411010523/en/main.pdf?idApp=UINPBA00004N&text.app=https://www.elsevier.es/" "EPUB" => "https://multimedia.elsevier.es/PublicationsMultimediaV1/item/epub/S1807593224000863?idApp=UINPBA00004N" ]
| Year/Month | Html | Total | |
|---|---|---|---|
| 2024 November | 12 | 3 | 15 |
| 2024 October | 89 | 53 | 142 |