Skip to content

Commit 06a0993

Browse files
authored
Merge pull request #21 from AKSW/dev
merge updates with 2 new ET papers, 2 updates and some keyword separator fixes
2 parents 984b74b + 8967ff9 commit 06a0993

1 file changed

Lines changed: 50 additions & 19 deletions

File tree

‎aksw.bib‎

Lines changed: 50 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -12682,16 +12682,22 @@ @Article{Brei2025Queryfyknowledgegraphs
1268212682
}
1268312683

1268412684
@InProceedings{Meyer2025LLMKGBench3,
12685-
author = {Lars-Peter Meyer and Johannes Frey and Desiree Heim and Felix Brei and Claus Stadler and Kurt Junghanns and Michael Martin},
12686-
title = {{LLM-KG-Bench} 3.0: A Compass for SemanticTechnology Capabilities in the Ocean of {LLMs}},
12687-
year = {2025},
12688-
note = {to appear in ESWC25 Resource Track Proceedings},
12689-
abstract = {Current Large Language Models (LLMs) can assist developing program code beside many other things, but can they support working with Knowledge Graphs (KGs) as well? Which LLM is offering the best capabilities in the field of Semantic Web and Knowledge Graph Engineering (KGE)? Is this possible to determine without checking many answers manually? The LLM-KG-Bench framework in Version 3.0 is designed to answer these questions. It consists of an extensible set of tasks for automated evaluation of LLM answers and covers different aspects of working with semantic technologies.
12685+
author = {Lars-Peter Meyer and Johannes Frey and Desiree Heim and Felix Brei and Claus Stadler and Kurt Junghanns and Michael Martin},
12686+
booktitle = {The Semantic Web},
12687+
title = {{LLM-KG-Bench} 3.0: A Compass for SemanticTechnology Capabilities in the Ocean of {LLMs}},
12688+
year = {2025},
12689+
address = {Cham},
12690+
editor = {Curry, Edward and Acosta, Maribel and Poveda-Villal{\'o}n, Maria and van Erp, Marieke and Ojo, Adegboyega and Hose, Katja and Shimizu, Cogan and Lisena, Pasquale},
12691+
pages = {280–296},
12692+
publisher = {Springer Nature Switzerland},
12693+
abstract = {Current Large Language Models (LLMs) can assist developing program code beside many other things, but can they support working with Knowledge Graphs (KGs) as well? Which LLM is offering the best capabilities in the field of Semantic Web and Knowledge Graph Engineering (KGE)? Is this possible to determine without checking many answers manually? The LLM-KG-Bench framework in Version 3.0 is designed to answer these questions. It consists of an extensible set of tasks for automated evaluation of LLM answers and covers different aspects of working with semantic technologies.
1269012694

1269112695
In this paper the LLM-KG-Bench framework is presented in Version 3 along with a dataset of prompts, answers and evaluations generated with it and several state-of-the-art LLMs. Significant enhancements have been made to the framework since its initial release, including an updated task API that offers greater flexibility in handling evaluation tasks, revised tasks, and extended support for various open models through the vllm library, among other improvements. A comprehensive dataset has been generated using more than 30 contemporary open and proprietary LLMs, enabling the creation of exemplary model cards that demonstrate the models’ capabilities in working with RDF and SPARQL, as well as comparing their performance on Turtle and JSON-LD RDF serialization tasks.},
12692-
doi = {10.48550/arXiv.2505.13098},
12693-
keywords = {group_aksw sys:relevantFor:infai es lpmeyer frey brei junghanns stadler martin},
12694-
url = {https://arxiv.org/abs/2505.13098},
12696+
doi = {10.1007/978-3-031-94578-6_16},
12697+
isbn = {9783031945786},
12698+
issn = {1611-3349},
12699+
keywords = {group_aksw sys:relevantFor:infai es lpmeyer frey brei junghanns stadler martin},
12700+
url = {http://dx.doi.org/10.1007/978-3-031-94578-6_16},
1269512701
}
1269612702

1269712703
@Article{Roters2025StahlDigitalOntologyBasedWorkflows,
@@ -12744,15 +12750,19 @@ @Article{Bekemeier2025AdvancingDigitalTransformation
1274412750
}
1274512751

1274612752
@InProceedings{Heim2025HowdoScalingLaws,
12747-
author = {Desiree Heim and Lars-Peter Meyer and Markus Schröder and Johannes Frey and Andreas Dengel},
12748-
title = {How do Scaling Laws Apply to Knowledge Graph Engineering Tasks? The Impact of Model Size on Large Language Model Performance},
12749-
year = {2025},
12750-
note = {to appear in the proceedings of workshop ELMKE @ ESWC 2025},
12751-
abstract = {When using Large Language Models (LLMs) to support Knowledge Graph Engineering (KGE), one of the first indications when searching for an appropriate model is its size. According to the scaling laws, larger models typically show higher capabilities. However, in practice, resource costs are also an important factor and thus it makes sense to consider the ratio between model performance and costs. The LLM-KG-Bench framework enables the comparison of LLMs in the context of KGE tasks and assesses their capabilities of understanding and producing KGs and KG queries. Based on a dataset created in an LLM-KG-Bench run covering 26 open state-of-the-art LLMs, we explore the model size scaling laws specific to KGE tasks. In our analyses, we assess how benchmark scores evolve between different model size categories. Additionally, we inspect how the general score development of single models and families of models correlates to their size.
12753+
author = {Desiree Heim and Lars-Peter Meyer and Markus Schröder and Johannes Frey and Andreas Dengel},
12754+
booktitle = {Joint Proceedings of the ESWC 2025 Workshops and Tutorialsco-located with 22nd Extended Semantic Web Conference (ESWC 2025)},
12755+
title = {How do Scaling Laws Apply to Knowledge Graph Engineering Tasks? The Impact of Model Size on Large Language Model Performance},
12756+
year = {2025},
12757+
editor = {M. Sabou and A. Harth and P. Lisena and E. Curry and B. Zhang and R. Alharbi and Y. He and G. Rehm and S. Schimmler and S. Dietze and N. Manola and A. Cimmino and N. Fornara and V. Rodríguez-Doncel and J. Domingue and A. Rettinger and D. Trilling and M. Grobelnik and C. d'Amato and V. Fionda and I. Tiddi and G. Tolomei},
12758+
series = {{CEUR} Workshop Proceedings},
12759+
volume = {3977},
12760+
abstract = {When using Large Language Models (LLMs) to support Knowledge Graph Engineering (KGE), one of the first indications when searching for an appropriate model is its size. According to the scaling laws, larger models typically show higher capabilities. However, in practice, resource costs are also an important factor and thus it makes sense to consider the ratio between model performance and costs. The LLM-KG-Bench framework enables the comparison of LLMs in the context of KGE tasks and assesses their capabilities of understanding and producing KGs and KG queries. Based on a dataset created in an LLM-KG-Bench run covering 26 open state-of-the-art LLMs, we explore the model size scaling laws specific to KGE tasks. In our analyses, we assess how benchmark scores evolve between different model size categories. Additionally, we inspect how the general score development of single models and families of models correlates to their size.
1275212761
Our analyses revealed that, with a few exceptions, the model size scaling laws generally also apply to the selected KGE tasks. However, in some cases, plateau or ceiling effects occurred, i.e., the task performance did not change much between a model and the next larger model. In these cases, smaller models could be considered to achieve high cost-effectiveness. Regarding models of the same family, sometimes smaller models performed worse than larger models of the same family. These effects occurred only locally. Hence it is advisable to additionally test the next smallest and largest model of the same family.},
12753-
doi = {10.48550/ARXIV.2505.16276},
12754-
keywords = {group_aksw sys:relevantFor:infai es lpmeyer frey},
12755-
url = {https://arxiv.org/pdf/2505.16276},
12762+
doi = {10.48550/ARXIV.2505.16276},
12763+
issn = {1613-0073},
12764+
keywords = {group_aksw sys:relevantFor:infai es lpmeyer frey},
12765+
url = {https://ceur-ws.org/Vol-3977/elmke-3.pdf},
1275612766
}
1275712767

1275812768
@Article{arndt-n-2025--webarchiv,
@@ -12767,7 +12777,7 @@ @Article{arndt-n-2025--webarchiv
1276712777
volume = {72},
1276812778
abstract = {The statutory collection mandate of the German National Library (DNB) includes collecting websites in the German web. The DNB has operated a web archive for this purpose since 2012. It is to be significantly expanded over the next few years. In order to accommodate this expansion, a new web archive system is currently under development and questions now need to be addressed regarding the selection of websites to be collected. The article discusses various aspects, including the special features of the web medium that need to be taken into account when making selections, the principle of representative diversity, and systematised selection approaches. It also outlines the stake­ holders, the basic requirements and the cornerstones of a new web archive system.},
1276912779
doi = {10.3196/186429502572237},
12770-
keywords = {arndt,group_aksw},
12780+
keywords = {arndt group_aksw},
1277112781
}
1277212782

1277312783
@InProceedings{Frey2025BreakingAccessibilityBarriers,
@@ -12779,7 +12789,7 @@ @InProceedings{Frey2025BreakingAccessibilityBarriers
1277912789
month = may,
1278012790
abstract = {This paper introduces a novel concept and implementation of an ontology proxy designed to seamlessly enhance accessibility and reliability of the Web of Ontologies by addressing challenges such as link rot, evolution inconsistencies, and communication failures. The proxy features a time travel mode, powered by DBpedia Archivo, that provides access to archived and versioned snapshots of ontologies. This enables failure recovery and the emulation of a consistent state in time, supporting reproducible research and enhancing the FAIRness of ontologies and associated (meta)data in a plug-and-play manner. Initial evaluations show significant improvements in ontology retrieval success rates, underscoring the proxy’s potential as a viable interface for breaking accessibility barriers.},
1278112791
doi = {https://dx.doi.org/10.1145/3701716.3715531},
12782-
keywords = {arndt,frey,group_aksw,kilt},
12792+
keywords = {arndt frey group_aksw kilt},
1278312793
}
1278412794

1278512795
@Article{arndt-n-2023--itit-0109,
@@ -12793,7 +12803,28 @@ @Article{arndt-n-2023--itit-0109
1279312803
pages = {139-141},
1279412804
volume = {65},
1279512805
doi = {10.1515/itit-2023-0109},
12796-
keywords = {arndt,usbeck,group_aksw},
12806+
keywords = {arndt usbeck group_aksw},
12807+
}
12808+
12809+
@InProceedings{Todorovikj2025CharacterizingKnowledgeGraph,
12810+
author = {Todorovikj, Sara and Meyer, Lars-Peter and Martin, Michael},
12811+
booktitle = {Proceedings of SEMANTiCS 2025 Poster Track},
12812+
title = {Characterizing Knowledge Graph Tasks in {LLM} Benchmarks Using Cognitive Complexity Frameworks},
12813+
year = {2025},
12814+
comment = {to appear at ceur-ws.org},
12815+
doi = {10.48550/arXiv.2509.19347},
12816+
keywords = {group_aksw sys:relevantFor:infai es lpmeyer martin},
12817+
url = {https://arxiv.org/pdf/2509.19347},
12818+
}
12819+
12820+
@InProceedings{Brei2025ARUQULAAnLLM,
12821+
author = {Brei, Felix and Bühmann, Lorenz and Frey, Johannes and Gerber, Daniel and Meyer, Lars-Peter and Stadler, Claus and Bulert, Kirill},
12822+
booktitle = {Proceedings of Text2SPARQL Workshop @ ESWC 2025},
12823+
title = {{ARUQULA} - An {LLM} based {Text2SPARQL} Approach using {ReAct} and Knowledge Graph Exploration Utilities},
12824+
year = {2025},
12825+
doi = {10.48550/arXiv.2510.02200},
12826+
keywords = {group_aksw sys:relevantFor:infai es brei buehmann frey gerber lpmeyer stadler},
12827+
url = {https://arxiv.org/pdf/2510.02200},
1279712828
}
1279812829

1279912830
@Comment{jabref-meta: databaseType:bibtex;}

0 commit comments

Comments
 (0)