diff --git a/src/fundus/publishers/uk/nature.py b/src/fundus/publishers/uk/nature.py index 7925ebbf8..e347b284f 100644 --- a/src/fundus/publishers/uk/nature.py +++ b/src/fundus/publishers/uk/nature.py @@ -17,7 +17,8 @@ class NatureParser(ParserProxy): class V1(BaseParser): - _summary_selector = CSSSelector("div.c-article-abstract p, p.c-article-abstract") + VALID_UNTIL = datetime.date(2026, 2, 1) # This date is the best guess + _summary_selector: XPath = CSSSelector("div.c-article-abstract p, p.c-article-abstract") _paragraph_selector = XPath( "//div[@data-test='access-teaser']//p" @@ -85,3 +86,32 @@ def images(self) -> List[Image]: author_selector=self._author_pattern, lower_boundary_selector=self._lower_boundary_selector, ) + + class V1_1(V1): + _paragraph_selector = XPath( + "//div[@data-test='main-content' or contains(@class,'main-content')]//p" + "[" + " not(ancestor::*[@data-label='Related' or contains(@class, 'recommended')])" + " and not(contains(@class, 'recommended__title'))" + " and not(ancestor::figure)" + " and not(ancestor::figcaption)" + " and not(ancestor::a)" + " and not(contains(@class, 'app-access-wall'))" + " and text()" + "] |" + "//div[@class='c-article-body']/section//p" + ) + _summary_selector = XPath("//div[@class='c-article-teaser-text']") + _subheadline_selector = XPath( + "//div[@data-test='main-content' or contains(@class,'main-content')]" + "//h2" + "[" + "not(ancestor::article[contains(@class, 'recommended')])" + " and not(contains(@class, 'app-access-wall'))" + " and not(@id='access-options')" + "] |" + "//div[@class='c-article-body']/section//h2" + ) + + _lower_boundary_selector = XPath("(//aside)[2]") + _paywall_selector = XPath("//div[contains(@class, 'buybox')]") diff --git a/tests/resources/parser/test_data/uk/Nature.json b/tests/resources/parser/test_data/uk/Nature.json index dde829134..780b6e18e 100644 --- a/tests/resources/parser/test_data/uk/Nature.json +++ b/tests/resources/parser/test_data/uk/Nature.json @@ -243,5 +243,184 @@ "Policy", "Politics" ] + }, + "V1_1": { + "authors": [ + "Rachel Brazil" + ], + "body": { + "summary": [ + "War and political unrest can force staff members and students to uproot their lives, exacting a heavy personal and professional toll." + ], + "sections": [ + { + "headline": [], + "paragraphs": [ + "“I will never forget that Saturday evening when I first saw Russian tanks on the streets of my city,” says Viktoriya Voropayeva, a systems engineer and vice-rector at the Donetsk National Technology University (DonNTU). In 2014, after Russian-backed forces took over Donetsk, the unofficial capital of Ukraine’s Donbas region, Voropayeva and many of her colleagues chose to leave, setting up their university in exile. “We hoped that it would be one semester or one academic year,” she says about the university’s relocation to Drohobych in western Ukraine. “Nobody thought that it could be forever.” Her family left with only their documents, family photos and their cat.", + "The university’s first new home was in Pokrovsk, a small city about 60 kilometres away from Donetsk, still in the Donbas region, where it already had a sister institution. About one-third of its students and staff members moved into three academic buildings and two dormitories. “Most of the teachers who stayed in Donetsk did so not because they supported the Donetsk People’s Republic [the separatist government created by Russia-backed paramilitaries in 2014], but because they could not find the strength to leave everything — homes, elderly parents, hospitals, schools,” says Voropayeva.", + "In April 2022, two months after Russia’s full-scale invasion of Ukraine, the university moved again. It went to Lutsk in northwestern Ukraine, to a building offered by Lutsk National Technical University. Then, seven months later, it moved to its current base in Drohobych in the Lviv region, about 1,050 km from its original home. The city council offered several buildings to turn into offices and classrooms. The space is much smaller than the facilities in Donetsk, but since the war started, most classes are now held online. DonNTU went from an 18,000-strong student body and more than 2,000 staff members in 2013, to 1,180 students and 116 staff members in 2024.", + "How the invasion of Ukraine is affecting Russian expat researchersThe European nation is not the only place where war or political unrest has forced universities and their staff members into exile. Others around the world, including in Sudan and Myanmar, have also had to relocate. Some institutions have instead moved teaching online and found new ways to reach students and faculty members. What unites scholars is a will to keep education and scholarship alive, retaining a sense of community and, in some cases, the hope that they can return and be part of a better future in their homelands.", + "Voropayeva says DonNTU continues to maintain close ties with local schools and the community in Donetsk, organizing webinars and courses for schoolchildren and teachers. But it, and other displaced universities in Ukraine, are now also serving local students from their new locations and recruiting local staff members. More than 50% of DonNTU first-year students are from the Lviv region." + ] + }, + { + "headline": [ + "Checkpoint challenges" + ], + "paragraphs": [ + "Illya Khadzhynov, an economist and vice-rector for scientific work, was last in Donetsk in July 2014, when his institution, Donetsk National University (DonNU) was taken over by the pro-Russian separatist government. Students and academics protested to the Ukrainian government. It authorized the university’s roughly 700-km move west to a former jewellery factory in Vinnytsia, in the west-central region of Ukraine, a building with no lecture theatres or laboratories. DonNU’s plight prompted an unofficial motto that “the university is not only the walls, it’s the people”, says Khadzhynov. But with support from international donors, including US$350,000 from the International Renaissance Foundation, a Ukrainian charity founded by Hungarian-born US philanthropist George Soros, DonNU refurbished the factory building and re-established a campus there, including labs for research and teaching.", + "Khadzhynov estimates that about half of the university’s 12,000 students moved to Vinnytsia. In 2016, the institution changed its name to Vasyl’ Stus Donetsk National University, honouring its alumnus Vasyl Stus, a poet who died in a labour camp after going on hunger strike after his arrest for anti-Soviet activity in 1980.", + "Serhii Radio, a chemistry researcher at DonNU, says that back in 2014, not everyone felt able to leave everything behind. Those who did took only “the most necessary things that you can carry in two hands”, he says. He was unable to take any lab equipment that might stand out at the checkpoints they had to pass, controlled by armed pro-Russian separatist groups. “They checked personal belongings, scrolled through the contents of smartphones, examined saved photos and music, and even reviewed social-media accounts,” he explains.", + "I fled the war in Ukraine. Now I work on ways to help the country’s soil healKhadzhynov says that the arrival of DonNU initially created tensions in Vinnytsia because the institution was larger and therefore had been allotted many more student places by the government than had local universities. But now, the institution is more integrated into the local community and DonNU students are mostly from Vinnytsia and neighbouring regions. Their current 650-strong staff includes 180 who have been displaced from other cities.", + "The war itself has brought the added problem of electricity cuts, which make it difficult for Radio to perform experiments for his studies using single-crystal X-ray diffraction. “An experiment needs more than eight hours, and this is a very long time for electricity,” he says. At DonNTU, Voropayeva says: “Laboratory equipment has been in storage since 2022 or was destroyed during shelling.” The university is gradually restoring its labs, and a pooled centre for collective use of scientific equipment enables Ukrainian academic institutions to share resources, sometimes on a fee-for-service basis. DonNTU plans to provide access to its mobile ‘makerspace’, a computing cluster and a 3D-modelling centre.", + "Faculty members and students from both former Donetsk institutions report problems in finding affordable accommodation in their new, smaller cities. Salaries have become catastrophically low — Voropayeva says that a full-time associate professor receives the equivalent of €300–350 (US$350–410) a month, which is comparable to pre-war wages but now buys considerably less because of rising prices.", + "Another challenge for both institutions has been the move from the heavily industrial Donbas, where many institutions excelled in applied science, to a region with a different industrial history. DonNTU had particular expertise in coal mining and metallurgy but is now shifting teaching and industrial collaboration to areas such as composite materials, chemicals and natural-gas extraction, which are more in demand in western Ukraine." + ] + }, + { + "headline": [ + "Russians in exile" + ], + "paragraphs": [ + "The war in Ukraine has also forced some Russian academics into exile. Art historian Philip Fedchin was a staff member at the Smolny College of Liberal Arts and Sciences in St Petersburg, Russia — a collaboration forged in 1997 between Bard College in Annandale-on-Hudson, New York, and St Petersburg State University. Smolny was the first liberal-arts college in Russia, awarding degrees from 2003 and following a broad multidisciplinary curriculum. But in June 2021, Bard College was declared an “undesirable” organization by the Russian prosecutor’s general office and all ties were cut.", + "“Everybody at the university in Smolny was shocked. It was considered the worst possible scenario, but it was just one of the few minor signs of what is going to come,” says Fedchin, referencing the looming war with Ukraine. Many faculty members and students left the country when the invasion of Ukraine started in February 2022, he adds, with more following that September, when Russia began its mobilization of reservists into the military. Fedchin left Russia in 2020 owing to his opposition to the country’s political direction. He relocated to Germany, joining Bard College Berlin, where he now works as a technology strategist.", + "In November 2022, Fedchin and his colleagues launched Smolny Beyond Borders, a university-in-exile initiative that enables many former Smolny faculty members to teach courses online with a similar ethos to the original Smolny College. So far, they have taught 2,585 students. Last semester, they ran 23 courses and now offer a two-year associate-degree programme accredited by Bard College.", + "Fedchin says that about 50% of its students taking non-degree courses are in Russia. Only displaced students are eligible to enrol for its degree programmes, and are taught by displaced faculty members located all over the world.", + "One of those is Andrei Rodin, a researcher in the history and philosophy of mathematics. He and his family relocated from Russia to France in March 2022, where, as well as teaching maths and statistics for Smolny Beyond Borders, he has a temporary teaching position at the University of Lorraine in Nancy.", + "The ambitions of Smolny Beyond Borders have now grown to supporting a broader community of students in exile from other parts of the world through two Bard partner organizations. One is the Global Higher Education Alliance for the 21st Century network, which provides opportunities for students to pursue learning, and the other is the Realizing Higher Education Access Program, a 12-month bridging programme intended to prepare refugee students for university. It is currently under way in Kenya, Jordan, Bangladesh and parts of East Africa. “My group has become much more international,” says Rodin. “I have students from Afghanistan, from Africa, from refugee camps.”" + ] + }, + { + "headline": [ + "Fleeing civil war in Sudan" + ], + "paragraphs": [ + "Gihad Ibrahim fled his home in the Sudanese city of Khartoum North soon after the civil war broke out in April 2023. Most of the population in affected regions escaped to other parts of the African country, or to neighbouring nations. A large number moved to Cairo, including Ibrahim, an engineer who taught at Mashreq University and Sudan University of Science and Technology. The two institutions both abandoned their main campuses in the Khartoum region.", + "The civil war has devastated higher education. “Some of the universities are still not working today,” says Ibrahim. Others, including the University of Khartoum, have adopted online teaching and have set up centres in regional universities to host exams and crucial practical training in subjects such as medicine.", + "Salaries for most academics were reduced to 60% of the original amount for the first 9–12 months of the war, although they have subsequently rebounded, says Ibrahim. Pointing to one positive, he describes how displaced medical students created medical camps to try to care for locals in those areas, even with the little experience they had.", + "Some private universities have fared better than public ones. This includes the private Mashreq University, which specializes in applied science, engineering and medicine. It served 10,000 students at three campuses in and around Khartoum that had to close when fighting came to the region — so the university had to adapt.", + "The idea was for Mashreq University to establish mini teaching centres elsewhere, says Ibrahim, forging collaborations with the Red Sea University in Port Sudan, a city less affected by the war, and renting space in Jeddah, Saudi Arabia. In Cairo, Mashreq University rented an entire unused college building. “It also had its own laboratories, mainly focusing on the engineering side,” Ibrahim says, adding: “We had to re-establish and buy new equipment for all the laboratories related to the medical field.” A smaller number of students fled to the United Arab Emirates, from where they can access lectures that are live-streamed from the Cairo campus.", + "A planned expansion of the medical college of Mashreq University into a university hospital has been abandoned, alongside its once-growing graduate faculty. Student numbers have also decreased by 45% owing to drop-outs and temporary suspensions of study. “Many students were not able to pay their fees because we had 300–400% inflation after the war and lots of people have lost their jobs,” says Ibrahim. The college has tried to provide scholarships where it can. After two-and-a-half years working this way, Ibrahim says with pride and determination that they are sustaining teaching “regardless of the difficult situation”.", + "Sudan’s disastrous war — and the science it is imperillingAlthough the war is still devastating the Darfur region of Sudan, fighting has stopped in the capital and the central regions. But Ibrahim says these areas are devastated. “My house has been hit by a bomb and everything inside was stolen, including the air conditioning, the fridges, even the clothes,” he says. Most of the universities in conflict zones have also been destroyed and looted.", + "Mashreq University plans to refurbish its campuses, starting with the main site in Khartoum. The Sudanese government is now telling universities to return to their campuses. But Ibrahim says people are unlikely to do so without basic infrastructure such as electricity, and with safety still an issue, he does not yet plan to return." + ] + }, + { + "headline": [ + "Military coups in Myanmar" + ], + "paragraphs": [ + "After the February 2021 military coup in Myanmar, activists launched a civil-disobedience movement (the Spring Revolution), which saw academics, administrators and students refuse to return to the now military-run public universities. In parts of the country not controlled by the military, academic institutions came together to form interim university councils (IUCs). “The estimate is that there’s about 200 of these for the different higher-education institutions across Myanmar, and some of these have re-formed and are granting their own degrees,” says Elizabeth Rhoads, who researches citizenship and displacement in and from Myanmar at the Centre for East and South-East Asian Studies at Lund University in Sweden. She has worked with several IUCs, set up by either student unions or faculty unions.", + "Rhoads says that reports from Myanmar indicate that these IUCs function in various ways. “Some interim university councils are fully functional medical and dental schools, while others could be two professors who might not even have been at the same university before but are from the same discipline or the same area of the country and have banded together.” She thinks that about 10% are granting degrees, but a United Nations report issued in June 2025 indicated that those educational certifications were often not recognized abroad.", + "Student and academic activists have also launched online universities, including the Spring University Myanmar (SUM) and the Virtual Federal University (VFU). “We saw that there was a gap to fill,” says the director of academic management at SUM, who asked to use the pseudonym James for security reasons. Established in May 2021, SUM has 11 schools, offering courses such as computer science, social science and human rights and public health. “We have on-boarded 25,000 registered students across all academic programmes,” James says. Students are from urban and rural areas; refugee and internally displaced-persons camps; and even some who have joined the revolutionary armed forces opposing the military government.", + "SUM received financial and technical support from several international aid organizations early on, but because this receded, most administrative staff work as volunteers. Many SUM professors are in Myanmar and are unable to leave, but they still receive payment from student fees; others teach from exile in Thailand or universities in other countries. James says that student tuition fees amount to the equivalent of US$95–109 for a nine-month diploma. Faculty members upload course material to a learning-management system and organize real-time online teaching. At one point, SUM offered some in-person vocational training in the border areas between Thailand and Myanmar — ranging from cooking and eco-printing to water filtration and basic computer skills — but logistical problems mean that these are now taught online.", + "Where Internet connectivity is unreliable, organizers have also distributed ‘SUM Boxes’ — palm-sized devices that can transmit a local hotspot so that students can access teaching material. “We have successfully delivered 38 boxes to the rural areas,” says James.", + "Rhoads and her colleague Matthew Walton, a Myanmar-politics researcher at the University of Toronto, Canada, have been teaching at VFU as part of a similar initiative, but one offering free tuition. They were also part of a team that developed a ‘capstone programme’ that enables students who had nearly completed degrees before the military coup to graduate, with one of the IUCs issuing their degree. The scheme, also supported by Lund University, provides the extra courses the students need, including core ones on research methods, and supervision of a thesis.", + "“We have a mixture of professors who are part of the civil-disobedience movement” inside Myanmar “that are doing supervision, and, of course, we are”, too, says Rhoads. The group is also looking at how to award credit for internships and the sorts of experiential learning that students have been doing since the coup. “Young people have stepped in as teachers, organizers, researchers — they are playing sometimes very senior roles in new organizations, and the question of how to recognize this experience is one that has to be worked out,” says Rhoads.", + "Both SUM and VFU now have students whose qualifications have been accepted for further study at other universities in Thailand and Hong Kong, although Walton says that there’s work to do to make sure that qualifications at these ‘institutions in exile’ are more widely accepted.", + "Public universities in the military-controlled regions of Myanmar are also running, with enrolment increasing since 2024 because of the ruling military junta’s army-conscription policy, which does not apply to university students, explains Rhoads. Walton says that this underscores the political importance of these new universities in exile: “asserting that they are the legitimate ones” with the right to grant degrees.", + "Beyond concerns around the safety of SUM and VFU professors still in Myanmar, another challenge is financial. In early 2025, the two institutions had grants from the US Agency for International Development (USAID) rescinded by the administration of US President Donald Trump as part of its freeze on foreign aid. Eighty per cent of USAID’s programmes were subsequently cancelled and the agency was closed in July 2025.", + "Rhoads concludes by saying that Myanmar’s new universities have a broader role for the future. “It’s necessary to have the wheel still turning, so that when a conflict is over, there’s something there to build on. This isn’t the band aid, this is the building block.”" + ] + } + ] + }, + "images": [ + { + "versions": [ + { + "url": "https://media.nature.com/w767/magazine-assets/d41586-025-04160-8/d41586-025-04160-8_52015120.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + } + ], + "is_cover": true, + "description": "A solitary solider walks through rubble on a road outside a badly damaged university building with columns", + "caption": "Donetsk National Technology University in Ukraine had to leave its second location in Pokrovsk after Russia’s full-scale invasion of Ukraine.", + "authors": [ + "Yan Dobronosov/Global Images Ukraine/Getty" + ], + "position": 258 + }, + { + "versions": [ + { + "url": "https://media.nature.com/lw767/magazine-assets/d41586-025-04160-8/d41586-025-04160-8_52015114.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + }, + { + "url": "https://media.nature.com/lw319/magazine-assets/d41586-025-04160-8/d41586-025-04160-8_52015114.jpg?as=webp", + "query_width": null, + "size": { + "width": 319, + "height": 0 + }, + "type": "image/webp" + }, + { + "url": "https://media.nature.com/lw767/magazine-assets/d41586-025-04160-8/d41586-025-04160-8_52015114.jpg?as=webp", + "query_width": null, + "size": { + "width": 767, + "height": 0 + }, + "type": "image/webp" + } + ], + "is_cover": false, + "description": "Portrait of Gihad Ibrahim", + "caption": "Engineer Gihad Ibrahim had to leave his home in the Sudanese city of Khartoum North when civil war broke out.", + "authors": [ + "Mashreq University" + ], + "position": 304 + }, + { + "versions": [ + { + "url": "https://media.nature.com/lw767/magazine-assets/d41586-025-04160-8/d41586-025-04160-8_52015112.jpg", + "query_width": null, + "size": null, + "type": "image/jpeg" + }, + { + "url": "https://media.nature.com/lw319/magazine-assets/d41586-025-04160-8/d41586-025-04160-8_52015112.jpg?as=webp", + "query_width": null, + "size": { + "width": 319, + "height": 0 + }, + "type": "image/webp" + }, + { + "url": "https://media.nature.com/lw767/magazine-assets/d41586-025-04160-8/d41586-025-04160-8_52015112.jpg?as=webp", + "query_width": null, + "size": { + "width": 767, + "height": 0 + }, + "type": "image/webp" + } + ], + "is_cover": false, + "description": "A young protester gestures angrily while sitting on a road, visible between the legs of riot police standing in a row", + "caption": "Anti-coup protesters blocked roads and sat in front of riot police in Yangon, Myanmar, after the February 2021 military coup.", + "authors": [ + "Panos Pictures" + ], + "position": 326 + } + ], + "publishing_date": "2026-02-06 00:00:00+00:00", + "title": "Universities in exile: displaced scholars count the costs of starting afresh", + "topics": [ + "Careers", + "Institutions", + "Lab life", + "Politics" + ] } } diff --git a/tests/resources/parser/test_data/uk/Nature_2026_02_07.html.gz b/tests/resources/parser/test_data/uk/Nature_2026_02_07.html.gz new file mode 100644 index 000000000..4a26981c6 Binary files /dev/null and b/tests/resources/parser/test_data/uk/Nature_2026_02_07.html.gz differ diff --git a/tests/resources/parser/test_data/uk/meta.info b/tests/resources/parser/test_data/uk/meta.info index f962083a2..72627731f 100644 --- a/tests/resources/parser/test_data/uk/meta.info +++ b/tests/resources/parser/test_data/uk/meta.info @@ -43,6 +43,10 @@ "url": "https://www.nature.com/articles/d41586-025-03571-x", "crawl_date": "2025-11-04 23:39:32.088596" }, + "Nature_2026_02_07.html.gz": { + "url": "https://www.nature.com/articles/d41586-025-04160-8", + "crawl_date": "2026-02-07 20:40:43.563662" + }, "TheGuardian_2024_02_24.html.gz": { "url": "https://www.theguardian.com/world/2024/feb/24/death-toll-in-valencia-fire-rises-to-10-as-remains-of-last-missing-person-found", "crawl_date": "2024-02-24 12:51:15.854319"