You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
{{ message }}
Repository navigation
Commit 06a0993
Browse filesBrowse the repository at this point in the historyBrowse files
author = {Lars-Peter Meyer and Johannes Frey and Desiree Heim and Felix Brei and Claus Stadler and Kurt Junghanns and Michael Martin},
12686
-
title = {{LLM-KG-Bench} 3.0: A Compass for SemanticTechnology Capabilities in the Ocean of {LLMs}},
12687
-
year = {2025},
12688
-
note = {to appear in ESWC25 Resource Track Proceedings},
12689
-
abstract = {Current Large Language Models (LLMs) can assist developing program code beside many other things, but can they support working with Knowledge Graphs (KGs) as well? Which LLM is offering the best capabilities in the field of Semantic Web and Knowledge Graph Engineering (KGE)? Is this possible to determine without checking many answers manually? The LLM-KG-Bench framework in Version 3.0 is designed to answer these questions. It consists of an extensible set of tasks for automated evaluation of LLM answers and covers different aspects of working with semantic technologies.
12685
+
author = {Lars-Peter Meyer and Johannes Frey and Desiree Heim and Felix Brei and Claus Stadler and Kurt Junghanns and Michael Martin},
12686
+
booktitle = {The Semantic Web},
12687
+
title = {{LLM-KG-Bench} 3.0: A Compass for SemanticTechnology Capabilities in the Ocean of {LLMs}},
12688
+
year = {2025},
12689
+
address = {Cham},
12690
+
editor = {Curry, Edward and Acosta, Maribel and Poveda-Villal{\'o}n, Maria and van Erp, Marieke and Ojo, Adegboyega and Hose, Katja and Shimizu, Cogan and Lisena, Pasquale},
12691
+
pages = {280–296},
12692
+
publisher = {Springer Nature Switzerland},
12693
+
abstract = {Current Large Language Models (LLMs) can assist developing program code beside many other things, but can they support working with Knowledge Graphs (KGs) as well? Which LLM is offering the best capabilities in the field of Semantic Web and Knowledge Graph Engineering (KGE)? Is this possible to determine without checking many answers manually? The LLM-KG-Bench framework in Version 3.0 is designed to answer these questions. It consists of an extensible set of tasks for automated evaluation of LLM answers and covers different aspects of working with semantic technologies.
12690
12694
12691
12695
In this paper the LLM-KG-Bench framework is presented in Version 3 along with a dataset of prompts, answers and evaluations generated with it and several state-of-the-art LLMs. Significant enhancements have been made to the framework since its initial release, including an updated task API that offers greater flexibility in handling evaluation tasks, revised tasks, and extended support for various open models through the vllm library, among other improvements. A comprehensive dataset has been generated using more than 30 contemporary open and proprietary LLMs, enabling the creation of exemplary model cards that demonstrate the models’ capabilities in working with RDF and SPARQL, as well as comparing their performance on Turtle and JSON-LD RDF serialization tasks.},
author = {Desiree Heim and Lars-Peter Meyer and Markus Schröder and Johannes Frey and Andreas Dengel},
12748
-
title = {How do Scaling Laws Apply to Knowledge Graph Engineering Tasks? The Impact of Model Size on Large Language Model Performance},
12749
-
year = {2025},
12750
-
note = {to appear in the proceedings of workshop ELMKE @ ESWC 2025},
12751
-
abstract = {When using Large Language Models (LLMs) to support Knowledge Graph Engineering (KGE), one of the first indications when searching for an appropriate model is its size. According to the scaling laws, larger models typically show higher capabilities. However, in practice, resource costs are also an important factor and thus it makes sense to consider the ratio between model performance and costs. The LLM-KG-Bench framework enables the comparison of LLMs in the context of KGE tasks and assesses their capabilities of understanding and producing KGs and KG queries. Based on a dataset created in an LLM-KG-Bench run covering 26 open state-of-the-art LLMs, we explore the model size scaling laws specific to KGE tasks. In our analyses, we assess how benchmark scores evolve between different model size categories. Additionally, we inspect how the general score development of single models and families of models correlates to their size.
12753
+
author = {Desiree Heim and Lars-Peter Meyer and Markus Schröder and Johannes Frey and Andreas Dengel},
12754
+
booktitle = {Joint Proceedings of the ESWC 2025 Workshops and Tutorialsco-located with 22nd Extended Semantic Web Conference (ESWC 2025)},
12755
+
title = {How do Scaling Laws Apply to Knowledge Graph Engineering Tasks? The Impact of Model Size on Large Language Model Performance},
12756
+
year = {2025},
12757
+
editor = {M. Sabou and A. Harth and P. Lisena and E. Curry and B. Zhang and R. Alharbi and Y. He and G. Rehm and S. Schimmler and S. Dietze and N. Manola and A. Cimmino and N. Fornara and V. Rodríguez-Doncel and J. Domingue and A. Rettinger and D. Trilling and M. Grobelnik and C. d'Amato and V. Fionda and I. Tiddi and G. Tolomei},
12758
+
series = {{CEUR} Workshop Proceedings},
12759
+
volume = {3977},
12760
+
abstract = {When using Large Language Models (LLMs) to support Knowledge Graph Engineering (KGE), one of the first indications when searching for an appropriate model is its size. According to the scaling laws, larger models typically show higher capabilities. However, in practice, resource costs are also an important factor and thus it makes sense to consider the ratio between model performance and costs. The LLM-KG-Bench framework enables the comparison of LLMs in the context of KGE tasks and assesses their capabilities of understanding and producing KGs and KG queries. Based on a dataset created in an LLM-KG-Bench run covering 26 open state-of-the-art LLMs, we explore the model size scaling laws specific to KGE tasks. In our analyses, we assess how benchmark scores evolve between different model size categories. Additionally, we inspect how the general score development of single models and families of models correlates to their size.
12752
12761
Our analyses revealed that, with a few exceptions, the model size scaling laws generally also apply to the selected KGE tasks. However, in some cases, plateau or ceiling effects occurred, i.e., the task performance did not change much between a model and the next larger model. In these cases, smaller models could be considered to achieve high cost-effectiveness. Regarding models of the same family, sometimes smaller models performed worse than larger models of the same family. These effects occurred only locally. Hence it is advisable to additionally test the next smallest and largest model of the same family.},
12753
-
doi = {10.48550/ARXIV.2505.16276},
12754
-
keywords = {group_aksw sys:relevantFor:infai es lpmeyer frey},
12755
-
url = {https://arxiv.org/pdf/2505.16276},
12762
+
doi = {10.48550/ARXIV.2505.16276},
12763
+
issn = {1613-0073},
12764
+
keywords = {group_aksw sys:relevantFor:infai es lpmeyer frey},
abstract = {The statutory collection mandate of the German National Library (DNB) includes collecting websites in the German web. The DNB has operated a web archive for this purpose since 2012. It is to be significantly expanded over the next few years. In order to accommodate this expansion, a new web archive system is currently under development and questions now need to be addressed regarding the selection of websites to be collected. The article discusses various aspects, including the special features of the web medium that need to be taken into account when making selections, the principle of representative diversity, and systematised selection approaches. It also outlines the stake holders, the basic requirements and the cornerstones of a new web archive system.},
abstract = {This paper introduces a novel concept and implementation of an ontology proxy designed to seamlessly enhance accessibility and reliability of the Web of Ontologies by addressing challenges such as link rot, evolution inconsistencies, and communication failures. The proxy features a time travel mode, powered by DBpedia Archivo, that provides access to archived and versioned snapshots of ontologies. This enables failure recovery and the emulation of a consistent state in time, supporting reproducible research and enhancing the FAIRness of ontologies and associated (meta)data in a plug-and-play manner. Initial evaluations show significant improvements in ontology retrieval success rates, underscoring the proxy’s potential as a viable interface for breaking accessibility barriers.},
12781
12791
doi = {https://dx.doi.org/10.1145/3701716.3715531},
0 commit comments