Merge pull request #111 from online-judge-tools/update-yukicoder

Update for yukicoder

Author: kmyk

onlinejudge/_implementation/testcase_zipper.py

@@ -41,7 +41,7 @@ def get(self) -> List[TestCase]:
         return self._testcases
 
 
-def extract_from_files(files: Iterator[Tuple[str, bytes]], format: str = '%s.%e', out: str = 'out') -> List[TestCase]:
+def extract_from_files(files: Iterator[Tuple[str, bytes]], format: str = '%s.%e', out: str = 'out', *, ignore_unmatched_samples: bool = False) -> List[TestCase]:
     """
     :param out: is the extension for output files. This is used when the zip-file contains files like `sample-1.ans` instead of `sample-1.out`.
     """
@@ -60,19 +60,20 @@ def extract_from_files(files: Iterator[Tuple[str, bytes]], format: str = '%s.%e'
     for name in sorted(names.keys()):
         data = names[name]
         if 'in' not in data or out not in data:
-            logger.error('dangling sample found: %s', str(data))
-            assert False
+            logger.error('unmatched sample found: %s', str(data))
+            if not ignore_unmatched_samples:
+                raise RuntimeError('unmatched sample found: {}'.format(data))
         else:
             testcases += [TestCase(name, *data['in'], *data[out])]
     return testcases
 
 
-def extract_from_zip(zip_data: bytes, format: str, out: str = 'out') -> List[TestCase]:
+def extract_from_zip(zip_data: bytes, format: str, out: str = 'out', *, ignore_unmatched_samples: bool = False) -> List[TestCase]:
     def iterate():
         with zipfile.ZipFile(io.BytesIO(zip_data)) as fh:
             for filename in fh.namelist():
                 if filename.endswith('/'):  # TODO: use `fh.getinfo(filename).is_dir()` after we stop supporting Python 3.5
                     continue
                 yield filename, fh.read(filename)
 
-    return extract_from_files(iterate(), format=format, out=out)
+    return extract_from_files(iterate(), format=format, out=out, ignore_unmatched_samples=ignore_unmatched_samples)

onlinejudge/_implementation/utils.py

@@ -72,6 +72,7 @@ def parse_content(parent: Union[bs4.NavigableString, bs4.Tag, bs4.Comment]) -> b
     return bs4.NavigableString(res)
 
 
+# TODO: send referer by default
 class FormSender:
     def __init__(self, form: bs4.Tag, url: str):
         assert isinstance(form, bs4.Tag)

onlinejudge/service/yukicoder.py

@@ -72,11 +72,18 @@ def _get_contests(cls, *, session: Optional[requests.Session] = None) -> List[Di
         session = session or utils.get_default_session()
         if cls._contests is None:
             cls._contests = []
-            for url in ('https://yukicoder.me/api/v1/contest/past', 'https://yukicoder.me/api/v1/contest/future'):
+            for tense in ('past', 'current', 'future'):
+                url = 'https://yukicoder.me/api/v1/contest/{}'.format(tense)
                 resp = utils.request('GET', url, session=session)
                 cls._contests.extend(json.loads(resp.content.decode()))
         return cls._contests
 
+    @classmethod
+    def _get_csrf_token(cls, *, session: requests.Session) -> str:
+        url = 'https://yukicoder.me/csrf_token'
+        resp = utils.request('GET', url, session=session)
+        return resp.content.decode()
+
 
 class YukicoderContest(onlinejudge.type.Contest):
     """
@@ -93,11 +100,10 @@ def list_problems(self, *, session: Optional[requests.Session] = None) -> Sequen
         """
 
         session = session or utils.get_default_session()
-        for contest in YukicoderService._get_contests(session=session):
-            if contest['Id'] == self.contest_id:
-                table = {problem['ProblemId']: problem['No'] for problem in YukicoderService._get_problems(session=session)}
-                return [YukicoderProblem(problem_no=table[problem_id]) for problem_id in contest['ProblemIdList']]
-        raise RuntimeError('Failed to get the contest information from API: {}'.format(self.get_url()))
+        url = 'https://yukicoder.me/api/v1/contest/id/{}'.format(self.contest_id)
+        resp = utils.request('GET', url, session=session)
+        data = json.loads(resp.content.decode())
+        return [YukicoderProblem(problem_id=problem_id) for problem_id in data['ProblemIdList']]
 
     def get_url(self) -> str:
         return 'https://yukicoder.me/contests/{}'.format(self.contest_id)
@@ -156,7 +162,7 @@ def download_system_cases(self, *, session: Optional[requests.Session] = None) -
         url = '{}/testcase.zip'.format(self.get_url())
         resp = utils.request('GET', url, session=session)
         fmt = 'test_%e/%s'
-        return onlinejudge._implementation.testcase_zipper.extract_from_zip(resp.content, fmt)
+        return onlinejudge._implementation.testcase_zipper.extract_from_zip(resp.content, fmt, ignore_unmatched_samples=True)  # NOTE: yukicoder's test sets sometimes contain garbages. The owner insists that this is an intended behavior, so we need to ignore them.
 
     def _parse_sample_tag(self, tag: bs4.Tag) -> Optional[Tuple[str, str]]:
         assert isinstance(tag, bs4.Tag)
@@ -177,6 +183,8 @@ def submit_code(self, code: bytes, language_id: LanguageId, *, filename: Optiona
         :raises NotLoggedInError:
         """
 
+        # NOTE: An implementation with the official API exists at 492d8d7. This is reverted at 2b7e6f5 because the API ignores cookies and says "提出するにはログインが必要です" at least at that time.
+
         session = session or utils.get_default_session()
         # get
         url = self.get_url() + '/submit'
@@ -192,7 +200,7 @@ def submit_code(self, code: bytes, language_id: LanguageId, *, filename: Optiona
         form.set('lang', language_id)
         form.set_file('file', filename or 'code', code)
         form.unset('custom_test')
-        resp = form.request(session=session)
+        resp = form.request(headers={'referer': url}, session=session)
         resp.raise_for_status()
         # result
         if 'submissions' in resp.url:
@@ -208,16 +216,10 @@ def submit_code(self, code: bytes, language_id: LanguageId, *, filename: Optiona
 
     def get_available_languages(self, *, session: Optional[requests.Session] = None) -> List[Language]:
         session = session or utils.get_default_session()
-        # get
-        # We use the problem page since it is available without logging in
-        resp = utils.request('GET', self.get_url(), session=session)
-        # parse
-        soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
-        select = soup.find('select', id='lang')
-        languages = []  # type: List[Language]
-        for option in select.find_all('option'):
-            languages += [Language(option.attrs['value'], ' '.join(option.string.split()))]
-        return languages
+        url = 'https://yukicoder.me/api/v1/languages'
+        resp = utils.request('GET', url, session=session)
+        data = json.loads(resp.content.decode())
+        return [Language(language['Id'], language['Name'] + ' (' + language['Ver'] + ')') for language in data]
 
     def get_url(self) -> str:
         if self.problem_no:

tests/get_problem_yukicoder.py

@@ -1,8 +1,10 @@
+import os
 import unittest
 
-import tests.utils
 from onlinejudge_api.main import main
 
+YUKICODER_TOKEN = os.environ.get('YUKICODER_TOKEN')
+
 
 class GetProblemYukicoderTest(unittest.TestCase):
     def test_100(self):
@@ -31,7 +33,7 @@ def test_100(self):
         actual = main(['get-problem', url], debug=True)
         self.assertEqual(expected, actual)
 
-    @unittest.skipIf(not tests.utils.is_logged_in('https://yukicoder.me/'), 'login is required')
+    @unittest.skipIf(YUKICODER_TOKEN is None, '$YUKICODER_TOKEN is required')
     def test_2_system(self):
         """This tests about system cases.
         """

tests/service_yukicoder.py

@@ -136,12 +136,12 @@ def test_from_url(self):
 
     def test_list_problems(self):
         self.assertEqual(YukicoderContest.from_url('https://yukicoder.me/contests/276').list_problems(), [
-            YukicoderProblem(problem_no=1168),
-            YukicoderProblem(problem_no=1169),
-            YukicoderProblem(problem_no=1170),
-            YukicoderProblem(problem_no=1171),
-            YukicoderProblem(problem_no=1172),
-            YukicoderProblem(problem_no=1173),
+            YukicoderProblem(problem_id=4401),
+            YukicoderProblem(problem_id=4809),
+            YukicoderProblem(problem_id=4387),
+            YukicoderProblem(problem_id=4729),
+            YukicoderProblem(problem_id=4271),
+            YukicoderProblem(problem_id=4255),
         ])
 
 

tests/submit_code_yukicoder.py

@@ -0,0 +1,50 @@
+import os
+import pathlib
+import tempfile
+import textwrap
+import unittest
+
+from onlinejudge_api.main import main
+
+YUKICODER_TOKEN = os.environ.get('YUKICODER_TOKEN')
+
+
+@unittest.skipIf(YUKICODER_TOKEN is None, '$YUKICODER_TOKEN is required')
+class SubmitYukicoderTest(unittest.TestCase):
+    def test_9000(self):
+        url = 'https://yukicoder.me/problems/no/9000'
+        filename = 'main.py'
+        code = textwrap.dedent(r"""
+            #!/usr/bin/env python3
+            print "Hello World!"
+        """)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            path = pathlib.Path(tempdir) / filename
+            with open(path, 'w') as fh:
+                fh.write(code)
+            language_id = main(['guess-language-id', '--file', str(path), url], debug=True)['result']['id']
+            data = main(['submit-code', '--file', str(path), '--language', language_id, url], debug=True)
+            self.assertEqual(data['status'], 'ok')
+
+    def test_527(self):
+        url = 'https://yukicoder.me/problems/527'
+        filename = 'main.cpp'
+        code = textwrap.dedent(r"""
+            #include <bits/stdc++.h>
+            using namespace std;
+            int main() {
+                int a, b; cin >> a >> b;
+                string s; cin >> s;
+                cout << a + b << ' ' << s << endl;
+                return 0;
+            }
+        """)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            path = pathlib.Path(tempdir) / filename
+            with open(path, 'w') as fh:
+                fh.write(code)
+            language_id = main(['guess-language-id', '--file', str(path), url], debug=True)['result']['id']
+            data = main(['submit-code', '--file', str(path), '--language', language_id, url], debug=True)
+            self.assertEqual(data['status'], 'ok')