|
1 | | -@article{jordan_evaluating_2019, |
2 | | - title={Evaluating Probabilistic Forecasts with scoringRules}, |
3 | | - volume={90}, |
4 | | - url={https://www.jstatsoft.org/index.php/jss/article/view/v090i12}, |
5 | | - doi={10.18637/jss.v090.i12}, |
6 | | - abstract={Probabilistic forecasts in the form of probability distributions over future events have become popular in several fields including meteorology, hydrology, economics, and demography. In typical applications, many alternative statistical models and data sources can be used to produce probabilistic forecasts. Hence, evaluating and selecting among competing methods is an important task. The scoringRules package for R provides functionality for comparative evaluation of probabilistic models based on proper scoring rules, covering a wide range of situations in applied work. This paper discusses implementation and usage details, presents case studies from meteorology and economics, and points to the relevant background literature.}, |
7 | | - number={12}, |
8 | | - journal={Journal of Statistical Software}, |
9 | | - author={Jordan, Alexander and Krüger, Fabian and Lerch, Sebastian}, |
10 | | - year={2019}, |
11 | | - pages={1-37} |
| 1 | +@inproceedings{AbadiEtAl2016, |
| 2 | + title={{TensorFlow}: a system for {Large-Scale} machine learning}, |
| 3 | + author={Abadi, Mart{\'\i}n and Barham, Paul and Chen, Jianmin and Chen, Zhifeng and Davis, Andy and Dean, Jeffrey and Devin, Matthieu and Ghemawat, Sanjay and Irving, Geoffrey and Isard, Michael and others}, |
| 4 | + booktitle={12th USENIX symposium on operating systems design and implementation (OSDI 16)}, |
| 5 | + pages={265--283}, |
| 6 | + year={2016} |
| 7 | +} |
| 8 | + |
| 9 | +@article{Allen2024, |
| 10 | + title={Weighted {scoringRules}: Emphasising particular outcomes when evaluating probabilistic forecasts}, |
| 11 | + author={Allen, Sam}, |
| 12 | + journal={Journal of Statistical Software}, |
| 13 | + pages={1--26}, |
| 14 | + year={2024}, |
| 15 | + doi={10.18637/jss.v110.i08} |
| 16 | +} |
| 17 | + |
| 18 | +@article{AraujoEtAl2025, |
| 19 | + title={Leveraging probabilistic forecasts for dengue preparedness and control: the 2024 {Dengue Forecasting Sprint} in {Brazil}}, |
| 20 | + author={Araujo, Eduardo Correa and Carvalho, Luiz Max and Ganem, Fabiana and Vacaro, Lua Bida and Bastos, Leonardo Soares and Freitas, Lais Picinini and Bastos, Marcio and Alencar, Ramila and Bianchi, Lucas and Capellan, Raul and others}, |
| 21 | + journal={medRxiv}, |
| 22 | + pages={2025--05}, |
| 23 | + year={2025}, |
| 24 | + doi={10.1101/2025.05.12.25327419}, |
| 25 | + publisher={Cold Spring Harbor Laboratory Press} |
| 26 | +} |
| 27 | + |
| 28 | +@article{Brier1950, |
| 29 | + title={Verification of forecasts expressed in terms of probability}, |
| 30 | + author={Brier, Glenn W}, |
| 31 | + journal={Monthly Weather Review}, |
| 32 | + volume={78}, |
| 33 | + pages={1--3}, |
| 34 | + year={1950} |
| 35 | +} |
| 36 | + |
| 37 | +@article{BrownEtAl2021, |
| 38 | + title={The {Model Evaluation Tools (MET)}: More than a decade of community-supported forecast verification}, |
| 39 | + author={Brown, Barbara and Jensen, Tara and Gotway, John Halley and Bullock, Randy and Gilleland, Eric and Fowler, Tressa and Newman, Kathryn and Adriaansen, Dan and Blank, Lindsay and Burek, Tatiana and others}, |
| 40 | + journal={Bulletin of the American Meteorological Society}, |
| 41 | + volume={102}, |
| 42 | + doi={10.1175/BAMS-D-19-0093.1}, |
| 43 | + pages={E782--E807}, |
| 44 | + year={2021} |
| 45 | +} |
| 46 | + |
| 47 | +@article{ChenEtAl2024, |
| 48 | + title = {Generative machine learning methods for multivariate ensemble postprocessing}, |
| 49 | + volume = {18}, |
| 50 | + doi = {10.1214/23-AOAS1784}, |
| 51 | + journal = {The Annals of Applied Statistics}, |
| 52 | + author = {Chen, Jieyu and Janke, Tim and Steinke, Florian and Lerch, Sebastian}, |
| 53 | + year = {2024}, |
| 54 | + pages = {159--183}, |
| 55 | +} |
| 56 | + |
| 57 | +@article{climpred, |
| 58 | + author = {Aaron Spring and others}, |
| 59 | + title = {climpred: A Python package for climate forecast verification and ensemble prediction}, |
| 60 | + journal = {Journal of Open Source Software}, |
| 61 | + volume = {6}, |
| 62 | + pages = {2970}, |
| 63 | + year = {2021}, |
| 64 | + doi = {10.21105/joss.02970}, |
| 65 | + url = {https://climpred.readthedocs.io/} |
| 66 | +} |
| 67 | + |
| 68 | +@article{DawidMusio2014, |
| 69 | + title={Theory and applications of proper scoring rules}, |
| 70 | + author={Dawid, Alexander Philip and Musio, Monica}, |
| 71 | + journal={Metron}, |
| 72 | + volume={72}, |
| 73 | + doi={10.1007/s40300-014-0039-y}, |
| 74 | + pages={169--183}, |
| 75 | + year={2014}, |
| 76 | + publisher={Springer} |
| 77 | +} |
| 78 | + |
| 79 | +@article{DawidEtAl2016, |
| 80 | + title={Minimum scoring rule inference}, |
| 81 | + author={Dawid, A Philip and Musio, Monica and Ventura, Laura}, |
| 82 | + journal={Scandinavian Journal of Statistics}, |
| 83 | + volume={43}, |
| 84 | + pages={123--138}, |
| 85 | + year={2016}, |
| 86 | + doi={10.1111/sjos.12168}, |
| 87 | + publisher={Wiley Online Library} |
| 88 | +} |
| 89 | + |
| 90 | +@article{Epstein1969, |
| 91 | + title={A scoring system for probability forecasts of ranked categories}, |
| 92 | + author={Epstein, Edward S}, |
| 93 | + journal={Journal of Applied Meteorology}, |
| 94 | + volume={8}, |
| 95 | + pages={985--987}, |
| 96 | + year={1969}, |
| 97 | + publisher={JSTOR} |
| 98 | +} |
| 99 | + |
| 100 | +@article{Ferro2014, |
| 101 | + title={Fair scores for ensemble forecasts}, |
| 102 | + author={Ferro, C A T}, |
| 103 | + journal={Quarterly Journal of the Royal Meteorological Society}, |
| 104 | + volume={140}, |
| 105 | + doi={10.1002/qj.2270}, |
| 106 | + pages={1917--1923}, |
| 107 | + year={2014}, |
| 108 | + publisher={Wiley Online Library} |
| 109 | +} |
| 110 | + |
| 111 | +@article{GneitingEtAl2005, |
| 112 | + title={Calibrated probabilistic forecasting using ensemble model output statistics and minimum {CRPS} estimation}, |
| 113 | + author={Gneiting, Tilmann and Raftery, Adrian E and Westveld, Anton H and Goldman, Tom}, |
| 114 | + journal={Monthly Weather Review}, |
| 115 | + volume={133}, |
| 116 | + pages={1098--1118}, |
| 117 | + year={2005}, |
| 118 | + publisher={American Meteorological Society}, |
| 119 | + doi={10.1175/MWR2904.1} |
| 120 | +} |
| 121 | + |
| 122 | +@article{GneitingRaftery2007, |
| 123 | + title={Strictly Proper Scoring Rules, Prediction, and Estimation}, |
| 124 | + author={Gneiting, T. and Raftery, A. E.}, |
| 125 | + journal={Journal of the American Statistical Association}, |
| 126 | + year={2007}, |
| 127 | + volume={102}, |
| 128 | + pages={359--378}, |
| 129 | + doi={10.1198/016214506000001437} |
| 130 | +} |
| 131 | + |
| 132 | +@article{Good1952, |
| 133 | + title={Rational Decisions}, |
| 134 | + author={Good, I. J.}, |
| 135 | + journal={Journal of the Royal Statistical Society B}, |
| 136 | + year={1952}, |
| 137 | + volume={14}, |
| 138 | + pages={107--114}, |
| 139 | + doi={10.1111/j.2517-6161.1952.tb00104.x}, |
| 140 | +} |
| 141 | + |
| 142 | +@article{HarrisEtAl2020, |
| 143 | + title={Array programming with NumPy}, |
| 144 | + author={Harris, Charles R and Millman, K Jarrod and Van Der Walt, St{\'e}fan J and Gommers, Ralf and Virtanen, Pauli and Cournapeau, David and Wieser, Eric and Taylor, Julian and Berg, Sebastian and Smith, Nathaniel J and others}, |
| 145 | + journal={Nature}, |
| 146 | + volume={585}, |
| 147 | + pages={357--362}, |
| 148 | + year={2020}, |
| 149 | + publisher={Nature Publishing Group UK London} |
| 150 | +} |
| 151 | + |
| 152 | +@article{HirschEtAl2024, |
| 153 | + title={Online Distributional Regression}, |
| 154 | + author={Hirsch, Simon and Berrisch, Jonathan and Ziel, Florian}, |
| 155 | + journal={arXiv preprint arXiv:2407.08750}, |
| 156 | + year={2024} |
| 157 | +} |
| 158 | + |
| 159 | +@article{Hirsch2025, |
| 160 | + title={Online Multivariate Regularized Distributional Regression for High-dimensional Probabilistic Electricity Price Forecasting}, |
| 161 | + author={Hirsch, Simon}, |
| 162 | + journal={arXiv preprint arXiv:2504.02518}, |
| 163 | + year={2025} |
| 164 | +} |
| 165 | + |
| 166 | +@software{JAX2018, |
| 167 | + author = {James Bradbury and Roy Frostig and Peter Hawkins and Matthew James Johnson and Chris Leary and Dougal Maclaurin and George Necula and Adam Paszke and Jake Vander{P}las and Skye Wanderman-{M}ilne and Qiao Zhang}, |
| 168 | + title = {{JAX}: composable transformations of {P}ython+{N}um{P}y programs}, |
| 169 | + url = {http://github.com/jax-ml/jax}, |
| 170 | + version = {0.3.13}, |
| 171 | + year = {2018}, |
| 172 | +} |
| 173 | + |
| 174 | +@article{JordanEtAl2019, |
| 175 | + title={Evaluating Probabilistic Forecasts with {scoringRules}}, |
| 176 | + author={Jordan, Alexander and Kr{\"u}ger, Fabian and Lerch, Sebastian}, |
| 177 | + journal={Journal of Statistical Software}, |
| 178 | + year={2019}, |
| 179 | + volume={90}, |
| 180 | + pages={1--37}, |
| 181 | + doi={10.18637/jss.v090.i12} |
| 182 | +} |
| 183 | + |
| 184 | +@article{LagoEtAl2021, |
| 185 | + title = {Forecasting day-ahead electricity prices: A review of state-of-the-art algorithms, best practices and an open-access benchmark}, |
| 186 | + journal = {Applied Energy}, |
| 187 | + volume = {293}, |
| 188 | + pages = {116983}, |
| 189 | + year = {2021}, |
| 190 | + doi = {10.1016/j.apenergy.2021.116983}, |
| 191 | + author = {Jesus Lago and Grzegorz Marcjasz and Bart {De Schutter} and Rafał Weron} |
| 192 | +} |
| 193 | + |
| 194 | +@inproceedings{LiuEtAl2025, |
| 195 | + title={Copula-based Probabilistic Prediction of Grid Frequency Dynamics}, |
| 196 | + author={Liu, Bolin and Koblenz, Maximilian and Grothe, Oliver}, |
| 197 | + booktitle={Proceedings of the 16th ACM International Conference on Future and Sustainable Energy Systems}, |
| 198 | + pages={733--741}, |
| 199 | + year={2025}, |
| 200 | + doi={10.1145/3679240.3734641}, |
| 201 | +} |
| 202 | + |
| 203 | +@article{LeeuwenburgEtAl2024, |
| 204 | + title={scores: A Python package for verifying and evaluating models and predictions with xarray}, |
| 205 | + author={Leeuwenburg, Tennessee and Loveday, Nicholas and Ebert, Elizabeth E and Cook, Harrison and Khanarmuei, Mohammadreza and Taggart, Robert J and Ramanathan, Nikeeth and Carroll, Maree and Chong, Stephanie and Griffiths, Aidan and others}, |
| 206 | + journal={arXiv preprint arXiv:2406.07817}, |
| 207 | + year={2024} |
| 208 | +} |
| 209 | + |
| 210 | +@article{LovedayEtAl2024, |
| 211 | + title={The {Jive} verification system and its transformative impact on weather forecasting operations}, |
| 212 | + author={Loveday, Nicholas and Griffiths, Deryn and Leeuwenburg, Tennessee and Taggart, Robert J and Pagano, Thomas C and Cheng, George and Plastow, Kevin and Ebert, Elizabeth E and Templeton, Cassandra and Carroll, Maree and others}, |
| 213 | + journal={Bulletin of the American Meteorological Society}, |
| 214 | + volume={105}, |
| 215 | + pages={E2047--E2063}, |
| 216 | + year={2024}, |
| 217 | + publisher={American Meteorological Society}, |
| 218 | + doi={10.1175/BAMS-D-23-0267.1} |
| 219 | +} |
| 220 | + |
| 221 | +@article{MathesonWinkler1976, |
| 222 | + title={Scoring rules for continuous probability distributions}, |
| 223 | + author={Matheson, James E and Winkler, Robert L}, |
| 224 | + journal={Management Science}, |
| 225 | + year={1976}, |
| 226 | + volume={22}, |
| 227 | + pages={1087--1096}, |
| 228 | + doi={10.1287/mnsc.22.10.1087} |
| 229 | +} |
| 230 | + |
| 231 | +@article{PacchiardiEtAl2024, |
| 232 | + title={Probabilistic forecasting with Generative Networks via Scoring Rule Minimization}, |
| 233 | + author={Pacchiardi, Lorenzo and Adewoyin, Rilwan A. and Dueben, Peter and Dutta, Ritabrata}, |
| 234 | + journal={Journal of Machine Learning Research}, |
| 235 | + volume={25}, |
| 236 | + year={2024}, |
| 237 | + pages={1--64}, |
| 238 | + url={jmlr.org/papers/v25/23-0038.html} |
| 239 | +} |
| 240 | + |
| 241 | +@inproceedings{PaszkeEtAl2019, |
| 242 | + title={{PyTorch}: An imperative style, high-performance deep learning library}, |
| 243 | + author={Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and others}, |
| 244 | + booktitle={Advances in Neural Information Processing Systems}, |
| 245 | + volume={32}, |
| 246 | + year={2019} |
| 247 | +} |
| 248 | + |
| 249 | +@software{properscoring, |
| 250 | + author = {The Climate Corporation}, |
| 251 | + title = {properscoring: Proper Scoring Rules in Python}, |
| 252 | + url = {https://pypi.python.org/pypi/properscoring}, |
| 253 | + version = {0.1}, |
| 254 | + year = {2015}, |
| 255 | +} |
| 256 | + |
| 257 | +@article{PulkkinenEtAl2019, |
| 258 | + title={Pysteps: An open-source {Python} library for probabilistic precipitation nowcasting (v1. 0)}, |
| 259 | + author={Pulkkinen, Seppo and Nerini, Daniele and P{\'e}rez Hortal, Andr{\'e}s A and Velasco-Forero, Carlos and Seed, Alan and Germann, Urs and Foresti, Loris}, |
| 260 | + journal={Geoscientific Model Development}, |
| 261 | + volume={12}, |
| 262 | + doi={10.5194/gmd-12-4185-2019}, |
| 263 | + pages={4185--4219}, |
| 264 | + year={2019}, |
| 265 | + publisher={Copernicus Publications G{\"o}ttingen, Germany} |
| 266 | +} |
| 267 | + |
| 268 | +@misc{pyforecasttools, |
| 269 | + author = {Steve Morley and contributors}, |
| 270 | + title = {PyForecastTools: Python tools for forecast verification and validation}, |
| 271 | + year = {2020}, |
| 272 | + url = {https://github.com/smorley/forecasttools}, |
| 273 | + note = {Accessed: 2025-06-24} |
| 274 | +} |
| 275 | + |
| 276 | +@article{ScheuererHamill2015, |
| 277 | + title={Variogram-based proper scoring rules for probabilistic forecasts of multivariate quantities}, |
| 278 | + author={Scheuerer, Michael and Hamill, Thomas M}, |
| 279 | + journal={Monthly Weather Review}, |
| 280 | + year={2015}, |
| 281 | + volume={143}, |
| 282 | + pages={1321--1334}, |
| 283 | + doi={10.1175/MWR-D-14-00269.1} |
| 284 | +} |
| 285 | + |
| 286 | +@article{TodterAhrens2012, |
| 287 | + title={Generalization of the ignorance score: Continuous ranked version and its decomposition}, |
| 288 | + author={T{\"o}dter, Julian and Ahrens, Bodo}, |
| 289 | + journal={Monthly Weather Review}, |
| 290 | + volume={140}, |
| 291 | + doi={10.1175/MWR-D-11-00266.1}, |
| 292 | + pages={2005--2017}, |
| 293 | + year={2012}, |
| 294 | + publisher={American Meteorological Society} |
| 295 | +} |
| 296 | + |
| 297 | +@article{xskillscore2020, |
| 298 | + author = {Ryan Abernathey and Julius Busecke and Deepak Cherian and others}, |
| 299 | + title = {xskillscore: Metrics for verifying and scoring forecasts and models}, |
| 300 | + journal = {Zenodo}, |
| 301 | + year = {2020}, |
| 302 | + doi = {10.5281/zenodo.3700574}, |
| 303 | + url = {https://github.com/xarray-contrib/xskillscore} |
| 304 | +} |
| 305 | + |
| 306 | +@article{WaghmareZiegel2025, |
| 307 | + title={Proper scoring rules for estimation and forecast evaluation}, |
| 308 | + author={Waghmare, Kartik and Ziegel, Johanna}, |
| 309 | + journal={arXiv preprint arXiv:2504.01781}, |
| 310 | + year={2025} |
| 311 | +} |
| 312 | + |
| 313 | +@article{Winkler1972, |
| 314 | + title={A decision-theoretic approach to interval estimation}, |
| 315 | + author={Winkler, Robert L}, |
| 316 | + journal={Journal of the American Statistical Association}, |
| 317 | + volume={67}, |
| 318 | + pages={187--191}, |
| 319 | + year={1972}, |
| 320 | + publisher={Taylor \& Francis} |
12 | 321 | } |
0 commit comments