From 9f7cdb4dea8eb7228c697c65a3b8c0b3fbad3176 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 29 Jun 2019 08:55:06 +0100 Subject: [PATCH 01/90] Use py36 environment in Tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 212aac4..2a7db7a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py32 +envlist = py26, py27, py36 [testenv] deps = nose From 810f7127e8a7eb43a4e425726074199abbf97699 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 29 Jun 2019 08:55:26 +0100 Subject: [PATCH 02/90] fix failing test (repr of duration class is different in python2 and latest python3) --- orgparse/date.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orgparse/date.py b/orgparse/date.py index 2600b19..260c339 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -476,8 +476,8 @@ def duration(self): >>> duration = OrgDateClock.from_str( ... 'CLOCK: [2010-08-08 Sun 17:00]--[2010-08-08 Sun 17:30] => 0:30' ... ).duration - >>> duration - datetime.timedelta(0, 1800) + >>> duration.seconds + 1800 >>> total_minutes(duration) 30.0 From 95d7a762069b543808837d64743158d9c9268f48 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 29 Jun 2019 08:56:02 +0100 Subject: [PATCH 03/90] Fix failing tests (StopIteration handling has to be explicit since python 3.7) --- orgparse/node.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/orgparse/node.py b/orgparse/node.py index bd3afc3..7516b55 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -796,7 +796,10 @@ def _iparse_sdc(self, ilines): They are assumed be in the first line. """ - line = next(ilines) + try: + line = next(ilines) + except StopIteration: + return (self._scheduled, self._deadline, self._closed) = parse_sdc(line) if not (self._scheduled or From d3d1b6e6344e3a125ddfc81f1ad426227e1c089c Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 29 Jun 2019 09:11:56 +0100 Subject: [PATCH 04/90] Handle StopIteration in _fin_children and add unit test for that --- orgparse/node.py | 5 ++++- orgparse/tests/data/02_tree_struct.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/orgparse/node.py b/orgparse/node.py index 7516b55..c8f0183 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -548,7 +548,10 @@ def parent(self): # FIXME: cache children nodes def _find_children(self): nodeiter = iter(self.env._nodes[self._index + 1:]) - node = next(nodeiter) + try: + node = next(nodeiter) + except StopIteration: + return if node.level <= self.level: return yield node diff --git a/orgparse/tests/data/02_tree_struct.py b/orgparse/tests/data/02_tree_struct.py index 17e876e..ae65fad 100644 --- a/orgparse/tests/data/02_tree_struct.py +++ b/orgparse/tests/data/02_tree_struct.py @@ -36,4 +36,6 @@ def nodedict(parent, children=[], previous=None, next=None): ('G6-H2',), ('G6-H2',), ('G6-H1', [], 'G6-H2'), + # G7 + (None, [], 'G6-H1'), ]] From 67e3df9cce91ab402447ded236f6ff34b9743abd Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 29 Jun 2019 09:20:05 +0100 Subject: [PATCH 05/90] Wrap next in _parse_pre in StopIteration (not sure if it ever happens) --- orgparse/node.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/orgparse/node.py b/orgparse/node.py index c8f0183..66012ee 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -769,7 +769,10 @@ def _parse_pre(self): self._parse_heading() # FIXME: make the following parsers "lazy" ilines = iter(self._lines) - next(ilines) # skip heading + try: + next(ilines) # skip heading + except StopIteration: + return ilines = self._iparse_sdc(ilines) ilines = self._iparse_clock(ilines) ilines = self._iparse_properties(ilines) From a34e9405e0d7bf7eaa01c00b9c7d047b025e3eb6 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 29 Jun 2019 09:24:11 +0100 Subject: [PATCH 06/90] Swap order of Sequence import to prevent warning on python3.5+ --- orgparse/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index 66012ee..052e73f 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1,9 +1,9 @@ import re import itertools try: - from collections import Sequence -except ImportError: from collections.abc import Sequence +except ImportError: + from collections import Sequence from .date import OrgDate, OrgDateClock, OrgDateRepeatedTask, parse_sdc from .inline import to_plain_text From 545f35d3e9ec205e32feb11bae5c0e1b03a7997e Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 29 Jun 2019 09:25:40 +0100 Subject: [PATCH 07/90] Add raw string markerts to RE_HEADING_STARS and RE_PROP to stop warnings --- orgparse/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index 052e73f..153cec9 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -37,7 +37,7 @@ def parse_heading_level(heading): if match: return (match.group(2), len(match.group(1))) -RE_HEADING_STARS = re.compile('^(\*+)\s*(.*?)\s*$') +RE_HEADING_STARS = re.compile(r'^(\*+)\s*(.*?)\s*$') def parse_heading_tags(heading): @@ -137,7 +137,7 @@ def parse_property(line): prop_val = int(h) * 60 + int(m) return (prop_key, prop_val) -RE_PROP = re.compile('^\s*:(.*?):\s*(.*?)\s*$') +RE_PROP = re.compile(r'^\s*:(.*?):\s*(.*?)\s*$') def parse_comment(line): From d55e02a5c529041ffd029f2c11467aef9aa46553 Mon Sep 17 00:00:00 2001 From: Evgeny Boykov Date: Mon, 7 Nov 2016 17:55:08 +1000 Subject: [PATCH 08/90] fix misprint --- README.rst | 2 +- orgparse/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index be978e2..ca59200 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ =========================================================== - orgparse - Pyton module for reading Emacs org-mode file + orgparse - Python module for reading Emacs org-mode file =========================================================== diff --git a/orgparse/__init__.py b/orgparse/__init__.py index cb3741d..0000db5 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -5,7 +5,7 @@ # ]]] """ =========================================================== - orgparse - Pyton module for reading Emacs org-mode file + orgparse - Python module for reading Emacs org-mode file =========================================================== From d8b250debbb5ab0d5753d416dad882031cd96654 Mon Sep 17 00:00:00 2001 From: Dmitrii Gerasimov Date: Mon, 1 Jul 2019 19:21:13 +0100 Subject: [PATCH 09/90] update links in readme --- README.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index ca59200..18b73c7 100644 --- a/README.rst +++ b/README.rst @@ -5,17 +5,15 @@ Links: -* `Documentation (at Read the Docs) `_ -* `Repository (at GitHub) `_ -* `Issue tracker (at GitHub) `_ -* `PyPI `_ -* `Travis CI `_ |build-status| +* `Documentation (at Read the Docs) `_ +* `Repository (at GitHub) `_ +* `Issue tracker (at GitHub) `_ +* `PyPI `_ +* `Travis CI `_ |build-status| .. |build-status| - image:: https://secure.travis-ci.org/tkf/orgparse.png?branch=master - :target: http://travis-ci.org/tkf/orgparse - :alt: Build Status - + image:: https://travis-ci.org/karlicoss/orgparse.svg?branch=master + :target: https://travis-ci.org/karlicoss/orgparse Install ------- From 398c45a72efffbe9ef516fecc7e027cd56b28f06 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 1 Jul 2019 19:45:28 +0100 Subject: [PATCH 10/90] add failing test for using iterator interface on node --- orgparse/tests/test_data.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index 857d5d4..58c4949 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -3,7 +3,7 @@ import pickle from nose.tools import eq_ -from .. import load +from .. import load, loads from ..utils.py3compat import execfile @@ -80,3 +80,18 @@ def check_picklable(dataname): def test_picklable(): for dataname in get_datanames(): yield (check_picklable, dataname) + + +def test_iter_node(): + root = loads(""" +* H1 +** H2 +*** H3 +* H4 +** H5 +""") + node = root[1] + assert node.heading == 'H1' + + by_iter = [n.heading for n in node] + assert by_iter == ['H1', 'H2', 'H3'] From 37260434df812f37981cb9e12ffb3f8f07625b86 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 1 Jul 2019 19:46:19 +0100 Subject: [PATCH 11/90] Fix __iter__ for OrgBaseNode. https://github.com/karlicoss/orgparse/issues/1 --- orgparse/node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/orgparse/node.py b/orgparse/node.py index 153cec9..ce81558 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -377,6 +377,8 @@ def __iter__(self): for node in self.env._nodes[self._index + 1:]: if node.level > level: yield node + else: + break def __len__(self): return sum(1 for _ in self) From e3b59588aa0dead740d69f60fb03ffa8d2270c75 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 1 Jul 2019 19:54:25 +0100 Subject: [PATCH 12/90] Prepare for 0.1 release --- orgparse/__init__.py | 2 +- setup.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 0000db5..4bef7c2 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -120,7 +120,7 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.0.1.dev3' +__version__ = '0.1.0' __author__ = 'Takafumi Arakaki' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] diff --git a/setup.py b/setup.py index dfcf8ad..3249ba3 100644 --- a/setup.py +++ b/setup.py @@ -22,14 +22,17 @@ long_description=orgparse.__doc__, keywords='org-mode, Emacs, parser', classifiers=[ - "Development Status :: 3 - Alpha", + 'Development Status :: 5 - Production/Stable' 'License :: OSI Approved :: BSD License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Text Processing :: Markup', # see: http://pypi.python.org/pypi?%3Aaction=list_classifiers ], ) From d3fcc0db1615af8a44b4ec71bf6e790385e35c69 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 1 Jul 2019 20:42:13 +0100 Subject: [PATCH 13/90] More preparation for 0.1.0 release --- Makefile | 26 ++++++++++++++++++++++++-- orgparse/__init__.py | 18 ++++++++---------- setup.py | 3 ++- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index b9ee208..3f0e52a 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,29 @@ cog: orgparse/__init__.py orgparse/__init__.py: README.rst cd orgparse && cog.py -r __init__.py +.PHONY: clean +clean: + rm -r dist/* + + +build: clean cog + python3 setup.py sdist bdist_wheel + +targets := $(wildcard dist/*) + +check: build $(targets) + twine check $(targets) + + + +## https://packaging.python.org/guides/using-testpypi +.PHONY: test-upload +test-upload: check $(targets) + twine upload --verbose --repository-url https://test.pypi.org/legacy/ $(targets) + ## Upload to PyPI -upload: cog - python setup.py register sdist upload +.PHONY: upload +upload: check $(target) + twine upload --verbose $(targets) + diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 4bef7c2..aadf36f 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -1,7 +1,7 @@ # Import README.rst using cog # [[[cog # from cog import out -# out('"""\n{0}\n"""'.format(file('../README.rst').read())) +# out('"""\n{0}\n"""'.format(open('../README.rst').read())) # ]]] """ =========================================================== @@ -11,17 +11,15 @@ Links: -* `Documentation (at Read the Docs) `_ -* `Repository (at GitHub) `_ -* `Issue tracker (at GitHub) `_ -* `PyPI `_ -* `Travis CI `_ |build-status| +* `Documentation (at Read the Docs) `_ +* `Repository (at GitHub) `_ +* `Issue tracker (at GitHub) `_ +* `PyPI `_ +* `Travis CI `_ |build-status| .. |build-status| - image:: https://secure.travis-ci.org/tkf/orgparse.png?branch=master - :target: http://travis-ci.org/tkf/orgparse - :alt: Build Status - + image:: https://travis-ci.org/karlicoss/orgparse.svg?branch=master + :target: https://travis-ci.org/karlicoss/orgparse Install ------- diff --git a/setup.py b/setup.py index 3249ba3..3ea9688 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +import setuptools from distutils.core import setup import orgparse @@ -22,7 +23,7 @@ long_description=orgparse.__doc__, keywords='org-mode, Emacs, parser', classifiers=[ - 'Development Status :: 5 - Production/Stable' + 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: BSD License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', From 5b59b5347a55999a694b3c62d04ff57f88a5c57f Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 17 Aug 2019 08:11:55 +0100 Subject: [PATCH 14/90] bump version to dev --- orgparse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index aadf36f..0058f5e 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -118,7 +118,7 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.1.0' +__version__ = '0.1.1dev1' __author__ = 'Takafumi Arakaki' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] From 05deaba1eeecc18fb59000bf08568c5532a87817 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 17 Aug 2019 08:12:38 +0100 Subject: [PATCH 15/90] Support range of time in DEADLINE/SCHEDULED --- orgparse/__init__.py | 2 +- orgparse/date.py | 40 ++++++++++++++++++--------- orgparse/tests/data/01_attributes.org | 3 ++ orgparse/tests/data/01_attributes.py | 15 +++++++++- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 0058f5e..674ac99 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -118,7 +118,7 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.1.1dev1' +__version__ = '0.1.1dev2' __author__ = 'Takafumi Arakaki' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] diff --git a/orgparse/date.py b/orgparse/date.py index 260c339..585f715 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -44,12 +44,13 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): >>> '{year}-{month}-{day}'.format(**m.groupdict()) '2010-06-21' >>> m = timestamp_re.match('<2005-10-01 Sat 12:30 +7m -3d>') - >>> m.groupdict() == { - ... 'year': '2005', 'month': '10', 'day': '01', - ... 'hour': '12', 'min': '30', - ... 'repeatpre': '+', 'repeatnum': '7', 'repeatdwmy': 'm', - ... 'warnpre': '-', 'warnnum': '3', 'warndwmy': 'd'} - True + >>> m.groupdict() + ... # doctest: +NORMALIZE_WHITESPACE + {'year': '2005', 'month': '10', 'day': '01', + 'hour': '12', 'min': '30', + 'end_hour': None, 'end_min': None, + 'repeatpre': '+', 'repeatnum': '7', 'repeatdwmy': 'm', + 'warnpre': '-', 'warnnum': '3', 'warndwmy': 'd'} When ``brtype = 'nobrace'``, cookie part cannot be retrieved. @@ -63,11 +64,11 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): >>> '{year}-{month}-{day}'.format(**m.groupdict()) '2010-06-21' >>> m = timestamp_re.match('2005-10-01 Sat 12:30 +7m -3d') - >>> m.groupdict() == { - ... 'year': '2005', 'month': '10', 'day': '01', - ... 'hour': '12', 'min': '30'} - True - + >>> m.groupdict() + ... # doctest: +NORMALIZE_WHITESPACE + {'year': '2005', 'month': '10', 'day': '01', + 'hour': '12', 'min': '30', + 'end_hour': None, 'end_min': None} """ if brtype == 'active': @@ -95,6 +96,11 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): ({ignore}+?) (?P<{prefix}hour>\d{{2}}) : (?P<{prefix}min>\d{{2}}) + ( # optional end time range + -- + (?P<{prefix}end_hour>\d{{2}}) : + (?P<{prefix}end_min>\d{{2}}) + )? )? """ regex_cookie = r""" @@ -418,8 +424,16 @@ def from_str(cls, string): match = cls._re.search(string) if match: mdict = match.groupdict() - return cls(cls._datetuple_from_groupdict(mdict), - active=cls._active_default) + start = cls._datetuple_from_groupdict(mdict) + end = None + end_hour = mdict['end_hour'] + end_min = mdict['end_min'] + if end_hour is not None and end_min is not None: + end_dict = {} + end_dict.update(mdict) + end_dict.update({'hour': end_hour, 'min': end_min}) + end = cls._datetuple_from_groupdict(end_dict) + return cls(start, end, active=cls._active_default) else: return cls(None) diff --git a/orgparse/tests/data/01_attributes.org b/orgparse/tests/data/01_attributes.org index 9d737d9..a6d7658 100644 --- a/orgparse/tests/data/01_attributes.org +++ b/orgparse/tests/data/01_attributes.org @@ -20,3 +20,6 @@ - <2010-08-16 Mon> DateList - <2010-08-07 Sat>--<2010-08-08 Sun> - <2010-08-09 Mon 00:30>--<2010-08-10 Tue 13:20> RangeList +* range in deadline +DEADLINE: <2019-09-06 Fri 10:00--11:20> + body diff --git a/orgparse/tests/data/01_attributes.py b/orgparse/tests/data/01_attributes.py index 498766c..6a0f508 100644 --- a/orgparse/tests/data/01_attributes.py +++ b/orgparse/tests/data/01_attributes.py @@ -38,4 +38,17 @@ body="", ) -data = [node1, node2, node1] +node3 = dict( + heading="range in deadline", + priority=None, + scheduled=OrgDate(None), + deadline=OrgDateDeadline((2019, 9, 6, 10, 0), (2019, 9, 6, 11, 20)), + closed=OrgDate(None), + clock=[], + properties={}, + datelist=[], + rangelist=[], + body=" body", +) + +data = [node1, node2, node1, node3] From 00880bdaf44a27bdd066f9ae1409d1029a0f6ed1 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 21 Aug 2019 08:19:23 +0100 Subject: [PATCH 16/90] Fix inline time ranges, support single dash in time range regex --- orgparse/date.py | 22 ++++++++++++++++------ orgparse/node.py | 4 +++- orgparse/tests/data/01_attributes.org | 2 ++ orgparse/tests/data/01_attributes.py | 4 +++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/orgparse/date.py b/orgparse/date.py index 585f715..7512fe4 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -97,7 +97,7 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): (?P<{prefix}hour>\d{{2}}) : (?P<{prefix}min>\d{{2}}) ( # optional end time range - -- + --? (?P<{prefix}end_hour>\d{{2}}) : (?P<{prefix}end_min>\d{{2}}) )? @@ -333,9 +333,18 @@ def _as_datetime(date): return date @staticmethod - def _datetuple_from_groupdict(dct, prefix=''): - keys = ['year', 'month', 'day', 'hour', 'min'] - return list(map(int, filter(None, (dct[prefix + k] for k in keys)))) + def _daterange_from_groupdict(dct, prefix=''): + start_keys = ['year', 'month', 'day', 'hour' , 'min'] + end_keys = ['year', 'month', 'day', 'end_hour', 'end_min'] + start_range = list(map(int, filter(None, (dct[prefix + k] for k in start_keys)))) + end_range = list(map(int, filter(None, (dct[prefix + k] for k in end_keys)))) + if len(end_range) < len(end_keys): + end_range = None + return (start_range, end_range) + + @classmethod + def _datetuple_from_groupdict(cls, dct, prefix=''): + return cls._daterange_from_groupdict(dct, prefix=prefix)[0] @classmethod def list_from_str(cls, string): @@ -350,7 +359,8 @@ def list_from_str(cls, string): [OrgDate((2012, 2, 10)), OrgDate((2012, 2, 12), None, False)] >>> OrgDate.list_from_str("this is not timestamp") [] - + >>> OrgDate.list_from_str("<2012-02-11 Sat 10:11--11:20>") + [OrgDate((2012, 2, 11, 10, 11, 0), (2012, 2, 11, 11, 20, 0))] """ match = TIMESTAMP_RE.search(string) if match: @@ -376,7 +386,7 @@ def list_from_str(cls, string): active=active) else: odate = cls( - cls._datetuple_from_groupdict(mdict, prefix), + *cls._daterange_from_groupdict(mdict, prefix), active=active) # FIXME: treat "repeater" and "warn" return [odate] + cls.list_from_str(rest) diff --git a/orgparse/node.py b/orgparse/node.py index ce81558..bccf2ec 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1186,11 +1186,13 @@ def rangelist(self): ... CLOCK: [2012-02-26 Sun 21:10]--[2012-02-26 Sun 21:15] => 0:05 ... Some inactive time range [2012-02-25 Sat]--[2012-02-27 Mon]. ... Some active time range <2012-02-26 Sun>--<2012-02-28 Tue>. + ... Some time interval <2012-02-27 Mon 11:23-12:10>. ... ''') >>> root.children[0].rangelist # doctest: +NORMALIZE_WHITESPACE [OrgDate((2012, 2, 25), (2012, 2, 28)), OrgDate((2012, 2, 25), (2012, 2, 27), False), - OrgDate((2012, 2, 26), (2012, 2, 28))] + OrgDate((2012, 2, 26), (2012, 2, 28)), + OrgDate((2012, 2, 27, 11, 23, 0), (2012, 2, 27, 12, 10, 0))] """ return self.get_timestamps(active=True, inactive=True, range=True) diff --git a/orgparse/tests/data/01_attributes.org b/orgparse/tests/data/01_attributes.org index a6d7658..8a047c5 100644 --- a/orgparse/tests/data/01_attributes.org +++ b/orgparse/tests/data/01_attributes.org @@ -9,6 +9,7 @@ - <2010-08-16 Mon> DateList - <2010-08-07 Sat>--<2010-08-08 Sun> - <2010-08-09 Mon 00:30>--<2010-08-10 Tue 13:20> RangeList + - <2019-08-10 Sat 16:30-17:30> TimeRange * A node without any attributed * DONE [#A] A node with a lot of attributes SCHEDULED: <2010-08-06 Fri> DEADLINE: <2010-08-10 Tue> CLOSED: [2010-08-08 Sun 18:00] @@ -20,6 +21,7 @@ - <2010-08-16 Mon> DateList - <2010-08-07 Sat>--<2010-08-08 Sun> - <2010-08-09 Mon 00:30>--<2010-08-10 Tue 13:20> RangeList + - <2019-08-10 Sat 16:30-17:30> TimeRange * range in deadline DEADLINE: <2019-09-06 Fri 10:00--11:20> body diff --git a/orgparse/tests/data/01_attributes.py b/orgparse/tests/data/01_attributes.py index 6a0f508..8359c26 100644 --- a/orgparse/tests/data/01_attributes.py +++ b/orgparse/tests/data/01_attributes.py @@ -18,11 +18,13 @@ rangelist=[ OrgDate((2010, 8, 7), (2010, 8, 8)), OrgDate((2010, 8, 9, 0, 30), (2010, 8, 10, 13, 20)), + OrgDate((2019, 8, 10, 16, 30, 0), (2019, 8, 10, 17, 30, 0)), ], body="""\ - <2010-08-16 Mon> DateList - <2010-08-07 Sat>--<2010-08-08 Sun> - - <2010-08-09 Mon 00:30>--<2010-08-10 Tue 13:20> RangeList""" + - <2010-08-09 Mon 00:30>--<2010-08-10 Tue 13:20> RangeList + - <2019-08-10 Sat 16:30-17:30> TimeRange""" ) node2 = dict( From 3ef88271b47f668b17c1d962ee9637c023b71cae Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 23 Aug 2019 07:42:31 +0100 Subject: [PATCH 17/90] Stabilize doctest by using sorted tuples instead of dict --- orgparse/date.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/orgparse/date.py b/orgparse/date.py index 7512fe4..320d617 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -44,13 +44,15 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): >>> '{year}-{month}-{day}'.format(**m.groupdict()) '2010-06-21' >>> m = timestamp_re.match('<2005-10-01 Sat 12:30 +7m -3d>') - >>> m.groupdict() + >>> from collections import OrderedDict + >>> sorted(m.groupdict().items()) ... # doctest: +NORMALIZE_WHITESPACE - {'year': '2005', 'month': '10', 'day': '01', - 'hour': '12', 'min': '30', - 'end_hour': None, 'end_min': None, - 'repeatpre': '+', 'repeatnum': '7', 'repeatdwmy': 'm', - 'warnpre': '-', 'warnnum': '3', 'warndwmy': 'd'} + [('day', '01'), + ('end_hour', None), ('end_min', None), + ('hour', '12'), ('min', '30'), + ('month', '10'), + ('repeatdwmy', 'm'), ('repeatnum', '7'), ('repeatpre', '+'), + ('warndwmy', 'd'), ('warnnum', '3'), ('warnpre', '-'), ('year', '2005')] When ``brtype = 'nobrace'``, cookie part cannot be retrieved. @@ -64,11 +66,12 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): >>> '{year}-{month}-{day}'.format(**m.groupdict()) '2010-06-21' >>> m = timestamp_re.match('2005-10-01 Sat 12:30 +7m -3d') - >>> m.groupdict() + >>> sorted(m.groupdict().items()) ... # doctest: +NORMALIZE_WHITESPACE - {'year': '2005', 'month': '10', 'day': '01', - 'hour': '12', 'min': '30', - 'end_hour': None, 'end_min': None} + [('day', '01'), + ('end_hour', None), ('end_min', None), + ('hour', '12'), ('min', '30'), + ('month', '10'), ('year', '2005')] """ if brtype == 'active': From 04dfefc6e93cd5062604483eb27a57fe9f3d9de7 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 23 Aug 2019 07:53:10 +0100 Subject: [PATCH 18/90] Bump version to 0.1.1 --- orgparse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 674ac99..2005363 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -118,7 +118,7 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.1.1dev2' +__version__ = '0.1.1' __author__ = 'Takafumi Arakaki' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] From 227672d7787e00976350c1252dfd0cc4458c902b Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 23 Aug 2019 07:59:06 +0100 Subject: [PATCH 19/90] fix make clean --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3f0e52a..45be100 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ orgparse/__init__.py: README.rst .PHONY: clean clean: - rm -r dist/* + rm -rf dist/* build: clean cog From cca4910a3be913325f5170ba8414ba5984e41161 Mon Sep 17 00:00:00 2001 From: Dmitrii Gerasimov Date: Wed, 18 Sep 2019 21:43:04 +0100 Subject: [PATCH 20/90] Create LICENSE add license file retroactively --- LICENSE | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fcdf7ba --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2012, Takafumi Arakaki +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 11168b85e4df7b3e84056afe43104b90b3888c3d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 18 Sep 2019 21:46:05 +0100 Subject: [PATCH 21/90] remove py26 env from tox (apparently it's reached EOL and gone from travis) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2a7db7a..c3e64b3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py36 +envlist = py27, py36 [testenv] deps = nose From 702faa6c05ff0642afc195288999b40f6897723c Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 18 Sep 2019 21:56:07 +0100 Subject: [PATCH 22/90] Update readme and setup file --- README.rst | 20 +++++++++----------- doc/source/conf.py | 7 +++++-- orgparse/__init__.py | 22 ++++++++++------------ setup.py | 10 ++++++++-- 4 files changed, 32 insertions(+), 27 deletions(-) diff --git a/README.rst b/README.rst index 18b73c7..3d29383 100644 --- a/README.rst +++ b/README.rst @@ -1,13 +1,10 @@ =========================================================== - orgparse - Python module for reading Emacs org-mode file + orgparse - Python module for reading Emacs org-mode files =========================================================== -Links: - -* `Documentation (at Read the Docs) `_ +* `Documentation (Read the Docs) `_ * `Repository (at GitHub) `_ -* `Issue tracker (at GitHub) `_ * `PyPI `_ * `Travis CI `_ |build-status| @@ -18,16 +15,17 @@ Links: Install ------- -You can install `orgparse` from PyPI_:: - pip install orgparse Usage ----- -Loading org object -^^^^^^^^^^^^^^^^^^ +There are pretty extensive doctests if you're interested in some specific method. Otherwise here are some example snippets: + + +Load org node +^^^^^^^^^^^^^ :: from orgparse import load, loads @@ -70,8 +68,8 @@ Traverse org tree * Heading 1 -Accessing to node attributes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Accessing node attributes +^^^^^^^^^^^^^^^^^^^^^^^^^ >>> root = loads(''' ... * DONE Heading :TAG: diff --git a/doc/source/conf.py b/doc/source/conf.py index ed2440c..222f2d5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -15,14 +15,17 @@ source_suffix = '.rst' master_doc = 'index' +# TODO not sure I'm doing that right.. +import orgparse + # General information about the project. project = u'orgparse' copyright = u'2012, Takafumi Arakaki' # The short X.Y version. -version = '0.0.1' +version = orgparse.__version__ # The full version, including alpha/beta/rc tags. -release = '0.0.1.dev3' +release = orgparse.__version__ exclude_patterns = [] diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 2005363..63d2797 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -5,15 +5,12 @@ # ]]] """ =========================================================== - orgparse - Python module for reading Emacs org-mode file + orgparse - Python module for reading Emacs org-mode files =========================================================== -Links: - -* `Documentation (at Read the Docs) `_ +* `Documentation (Read the Docs) `_ * `Repository (at GitHub) `_ -* `Issue tracker (at GitHub) `_ * `PyPI `_ * `Travis CI `_ |build-status| @@ -24,16 +21,17 @@ Install ------- -You can install `orgparse` from PyPI_:: - pip install orgparse Usage ----- -Loading org object -^^^^^^^^^^^^^^^^^^ +There are pretty extensive doctests if you're interested in some specific method. Otherwise here are some example snippets: + + +Load org node +^^^^^^^^^^^^^ :: from orgparse import load, loads @@ -76,8 +74,8 @@ * Heading 1 -Accessing to node attributes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Accessing node attributes +^^^^^^^^^^^^^^^^^^^^^^^^^ >>> root = loads(''' ... * DONE Heading :TAG: @@ -118,7 +116,7 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.1.1' +__version__ = '0.1.2dev0' __author__ = 'Takafumi Arakaki' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] diff --git a/setup.py b/setup.py index 3ea9688..b943662 100644 --- a/setup.py +++ b/setup.py @@ -15,13 +15,19 @@ package_data={ 'orgparse.tests.data': ['*.org'], }, + author=orgparse.__author__, author_email='aka.tkf@gmail.com', - url='https://github.com/tkf/orgparse', + maintainer='Dima Gerasimov (@karlicoss)', + maintainer_email='karlicoss@gmail.com', + + url='https://github.com/karlicoss/orgparse', license=orgparse.__license__, + description='orgparse - Emacs org-mode parser in Python', long_description=orgparse.__doc__, - keywords='org-mode, Emacs, parser', + + keywords='org org-mode emacs', classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: BSD License', From 3faf38a46b5a96d05a857b2995a5de72d2ab5371 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 21 Sep 2019 16:48:11 +0100 Subject: [PATCH 23/90] move tests to pytest --- orgparse/tests/test_data.py | 24 +++++++----------------- orgparse/tests/test_hugedata.py | 5 +---- tox.ini | 5 ++--- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index 58c4949..c7cb8e9 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -1,11 +1,11 @@ import os from glob import glob import pickle -from nose.tools import eq_ from .. import load, loads from ..utils.py3compat import execfile +import pytest # type: ignore DATADIR = os.path.join(os.path.dirname(__file__), 'data') @@ -47,7 +47,8 @@ def get_datanames(): yield os.path.splitext(os.path.basename(oname))[0] -def check_data(dataname): +@pytest.mark.parametrize('dataname', get_datanames()) +def test_data(dataname): """ Compare parsed data from 'data/*.org' and its correct answer 'data/*.py' """ @@ -58,29 +59,18 @@ def check_data(dataname): for (i, (node, kwds)) in enumerate(zip(root[1:], data)): for key in kwds: val = value_from_data_key(node, key) - eq_(kwds[key], val, - msg=('check value of {0}-th node of key "{1}" from "{2}".' - '\n\nParsed:\n{3}\n\nReal:\n{4}' - ).format(i, key, dataname, val, kwds[key])) + assert kwds[key] == val, 'check value of {0}-th node of key "{1}" from "{2}".\n\nParsed:\n{3}\n\nReal:\n{4}'.format(i, key, dataname, val, kwds[key]) - eq_(root.env.filename, oname) + assert root.env.filename == oname -def test_data(): - for dataname in get_datanames(): - yield (check_data, dataname) - - -def check_picklable(dataname): +@pytest.mark.parametrize('dataname', get_datanames()) +def test_picklable(dataname): oname = data_path(dataname, "org") root = load(oname) pickle.dumps(root) -def test_picklable(): - for dataname in get_datanames(): - yield (check_picklable, dataname) - def test_iter_node(): root = loads(""" diff --git a/orgparse/tests/test_hugedata.py b/orgparse/tests/test_hugedata.py index b72e27f..171d5b2 100644 --- a/orgparse/tests/test_hugedata.py +++ b/orgparse/tests/test_hugedata.py @@ -3,8 +3,6 @@ except ImportError: import pickle -from nose.tools import eq_ - from .. import loadi @@ -31,6 +29,5 @@ def test_picklable(): depth = 3 nodes_per_level = 1 root = loadi(generate_org_lines(num, depth, nodes_per_level)) - eq_(sum(1 for _ in root), - num_generate_org_lines(num, depth, nodes_per_level) + 1) + assert sum(1 for _ in root) == num_generate_org_lines(num, depth, nodes_per_level) + 1 pickle.dumps(root) # should not fail diff --git a/tox.ini b/tox.ini index c3e64b3..a1cc1a1 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,5 @@ envlist = py27, py36 [testenv] deps = - nose -commands = nosetests --with-doctest orgparse -changedir = {envtmpdir} + pytest +commands = pytest --doctest-modules --ignore-glob='**/_py3compat.py' orgparse From 3f65140041c4090cc13b81be2023b907bcbc23ce Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 21 Sep 2019 17:02:01 +0100 Subject: [PATCH 24/90] Support for Unicode in tags (resolves https://github.com/karlicoss/orgparse/issues/6) --- orgparse/node.py | 3 ++- orgparse/tests/data/05_tags.org | 1 + orgparse/tests/data/05_tags.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/orgparse/node.py b/orgparse/node.py index bccf2ec..b1842df 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -71,7 +71,8 @@ def parse_heading_tags(heading): tags = [] return (heading, tags) -RE_HEADING_TAGS = re.compile(r'(.*?)\s*:([a-zA-Z0-9@_:]+):\s*$') +# Tags are normal words containing letters, numbers, ‘_’, and ‘@’. https://orgmode.org/manual/Tags.html +RE_HEADING_TAGS = re.compile(r'(.*?)\s*:([\w@:]+):\s*$') def parse_heading_todos(heading, todo_candidates): diff --git a/orgparse/tests/data/05_tags.org b/orgparse/tests/data/05_tags.org index 9e6ca2a..651d7e0 100644 --- a/orgparse/tests/data/05_tags.org +++ b/orgparse/tests/data/05_tags.org @@ -6,3 +6,4 @@ * Node 5 :@_: * Node 6 :_tag_: * Heading: :with:colon: :tag: +* unicode :ёж:tag:háček: diff --git a/orgparse/tests/data/05_tags.py b/orgparse/tests/data/05_tags.py index 85a4db2..086512e 100644 --- a/orgparse/tests/data/05_tags.py +++ b/orgparse/tests/data/05_tags.py @@ -16,4 +16,6 @@ def nodedict(i, tags): [["_tag_"]], ])] + [ dict(heading='Heading: :with:colon:', tags=set(["tag"])), + ] + [ + dict(heading='unicode', tags=set(['ёж', 'tag', 'háček'])), ] From 655bb7648d4e848c9052afc2ee51dad3f2991544 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 21 Sep 2019 17:17:43 +0100 Subject: [PATCH 25/90] Fix broken python2 test --- orgparse/node.py | 2 +- orgparse/tests/data/05_tags.py | 2 ++ orgparse/tests/test_data.py | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index b1842df..9f946d2 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -71,7 +71,7 @@ def parse_heading_tags(heading): tags = [] return (heading, tags) -# Tags are normal words containing letters, numbers, ‘_’, and ‘@’. https://orgmode.org/manual/Tags.html +# Tags are normal words containing letters, numbers, '_', and '@'. https://orgmode.org/manual/Tags.html RE_HEADING_TAGS = re.compile(r'(.*?)\s*:([\w@:]+):\s*$') diff --git a/orgparse/tests/data/05_tags.py b/orgparse/tests/data/05_tags.py index 086512e..52aee63 100644 --- a/orgparse/tests/data/05_tags.py +++ b/orgparse/tests/data/05_tags.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + def nodedict(i, tags): return dict( heading="Node {0}".format(i), diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index c7cb8e9..44dd853 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -3,7 +3,7 @@ import pickle from .. import load, loads -from ..utils.py3compat import execfile +from ..utils.py3compat import execfile, PY3 import pytest # type: ignore @@ -52,6 +52,11 @@ def test_data(dataname): """ Compare parsed data from 'data/*.org' and its correct answer 'data/*.py' """ + if dataname == '05_tags': + if not PY3: + # python2 is end of life, so not worth fixing properly + pytest.skip('Ignoring test involving unicode') + oname = data_path(dataname, "org") data = load_data(data_path(dataname, "py")) root = load(oname) From 35c06285f52ac5eb59984640b7f31a4796d71c83 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 21 Sep 2019 17:33:41 +0100 Subject: [PATCH 26/90] add test example by @alphapapa --- orgparse/tests/test_misc.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 orgparse/tests/test_misc.py diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py new file mode 100644 index 0000000..65be28a --- /dev/null +++ b/orgparse/tests/test_misc.py @@ -0,0 +1,31 @@ +from .. import load, loads + +def test_stars(): + # https://github.com/karlicoss/orgparse/issues/7#issuecomment-533732660 + root = loads(""" +* Heading with text (A) + +The following line is not a heading, because it begins with a +star but has no spaces afterward, just a newline: + +* + +** Subheading with text (A1) + +This subheading is a child of (A). + +The next heading has no text, but it does have a space after +the star, so it's a heading: + +* + +This text is under the "anonymous" heading above, which would be (B). + +** Subheading with text (B1) + +This subheading is a child of the "anonymous" heading (B), not of heading (A). + """) + [h1, h2] = root.children + assert h1.heading == 'Heading with text (A)' + assert h2.heading == '' + From f87841bb434971b55464ece622688d76db7e247f Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 21 Sep 2019 17:43:25 +0100 Subject: [PATCH 27/90] Fix regex for extracting outline level https://github.com/karlicoss/orgparse/issues/7 --- orgparse/node.py | 4 +++- orgparse/tests/test_misc.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/orgparse/node.py b/orgparse/node.py index 9f946d2..a725c38 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -30,6 +30,8 @@ def parse_heading_level(heading): ('Heading', 1) >>> parse_heading_level('******** Heading') ('Heading', 8) + >>> parse_heading_level('*') # None since no space after star + >>> parse_heading_level('*bold*') # None >>> parse_heading_level('not heading') # None """ @@ -37,7 +39,7 @@ def parse_heading_level(heading): if match: return (match.group(2), len(match.group(1))) -RE_HEADING_STARS = re.compile(r'^(\*+)\s*(.*?)\s*$') +RE_HEADING_STARS = re.compile(r'^(\*+)\s+(.*?)\s*$') def parse_heading_tags(heading): diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index 65be28a..31584a8 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -12,6 +12,10 @@ def test_stars(): ** Subheading with text (A1) +*this_is_just* + + *some_bold_text* + This subheading is a child of (A). The next heading has no text, but it does have a space after From cd007b8f234ef6c99ca646d6a2622f16afafe419 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 21 Sep 2019 17:48:34 +0100 Subject: [PATCH 28/90] bump version --- orgparse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 63d2797..9f37dce 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -116,7 +116,7 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.1.2dev0' +__version__ = '0.1.2' __author__ = 'Takafumi Arakaki' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] From 7a459129d91ceb1ea23a000201ff40ee4aa713d5 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sat, 21 Sep 2019 17:51:46 +0100 Subject: [PATCH 29/90] bump version to dev --- orgparse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 9f37dce..da57422 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -116,7 +116,7 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.1.2' +__version__ = '0.1.3dev0' __author__ = 'Takafumi Arakaki' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] From d10a73b5963d3f6e38a3541647749c9ad2f0e756 Mon Sep 17 00:00:00 2001 From: John Lee Date: Sun, 1 Mar 2020 19:26:26 +0000 Subject: [PATCH 30/90] Add a few tests about comments Second one is failing --- orgparse/tests/test_data.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index 44dd853..8f5f6c6 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -90,3 +90,42 @@ def test_iter_node(): by_iter = [n.heading for n in node] assert by_iter == ['H1', 'H2', 'H3'] + + +def test_commented_headings_do_not_appear_as_children(): + root = loads(""" +* H1 +#** H2 +** H3 +#* H4 +#** H5 +* H6 +""") + top_level = root.children + assert len(top_level) == 2 + + h1 = top_level[0] + assert h1.heading == "H1" + assert h1.get_body() == "#** H2" + + [h3] = h1.children + assert h3.heading == "H3" + assert h3.get_body() == "#* H4\n#** H5" + + h6 = top_level[1] + assert h6.heading == "H6" + assert len(h6.children) == 0 + + +def test_commented_clock_entries_are_ignored_by_node_clock(): + root = loads(""" +* Heading +# * Floss +# SCHEDULED: <2019-06-22 Sat 08:30 .+1w> +# :LOGBOOK: +# CLOCK: [2019-06-04 Tue 16:00]--[2019-06-04 Tue 17:00] => 1:00 +# :END: +""") + [node] = root.children[0] + assert node.heading == "Heading" + assert node.clock == [] From 4ce88c3051ad5c3cbcc98150f49377da4fc0dfc6 Mon Sep 17 00:00:00 2001 From: John Lee Date: Sun, 1 Mar 2020 19:40:55 +0000 Subject: [PATCH 31/90] Don't parse clock expressions that are commented out Presumably there are also other org expressions that this code will still find clocks in that are not really there. For example (note leading whitespace): # CLOCK: [2019-06-04 Tue 16:00]--[2019-06-04 Tue 17:00] => 1:00 The above is not commented (because this form of comment must start at the first character on a line), so this fix still leaves that treated as a clock expression. I imagine org does not treat it as such, so that's probably a bad thing -- but it's unchanged behaviour and I'm only trying here to fix the case of explicitly commented code. --- orgparse/date.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/date.py b/orgparse/date.py index 320d617..f00d1f0 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -551,7 +551,7 @@ def from_str(cls, line): ) _re = re.compile( - r'CLOCK:\s+' + r'^(?!#).*CLOCK:\s+' r'\[(\d+)\-(\d+)\-(\d+)[^\]\d]*(\d+)\:(\d+)\]--' r'\[(\d+)\-(\d+)\-(\d+)[^\]\d]*(\d+)\:(\d+)\]\s+=>\s+(\d+)\:(\d+)' ) From cab168f0ae59a0bcfb9045ed505fed76d531a98c Mon Sep 17 00:00:00 2001 From: John Lee Date: Sun, 1 Mar 2020 20:02:13 +0000 Subject: [PATCH 32/90] Another failing test, this time for commented SCHEDULED: markers --- orgparse/tests/test_data.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index 8f5f6c6..7803bec 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -129,3 +129,13 @@ def test_commented_clock_entries_are_ignored_by_node_clock(): [node] = root.children[0] assert node.heading == "Heading" assert node.clock == [] + + +def test_commented_scheduled_marker_is_ignored_by_node_scheduled(): + root = loads(""" +* Heading +# SCHEDULED: <2019-06-22 Sat 08:30 .+1w> +""") + [node] = root.children[0] + assert node.heading == "Heading" + assert node.scheduled.start is None From 357f9419f126a5290d0e1265c958ff05ce0df965 Mon Sep 17 00:00:00 2001 From: John Lee Date: Sun, 1 Mar 2020 20:17:36 +0000 Subject: [PATCH 33/90] Don't parse SCHEDULED/DEADLINE/CLOSED markers that are commented out Same caveats as previous fix --- orgparse/date.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/date.py b/orgparse/date.py index f00d1f0..1cce7bf 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -421,7 +421,7 @@ def from_str(cls, string): def compile_sdc_re(sdctype): brtype = 'inactive' if sdctype == 'CLOSED' else 'active' return re.compile( - r'{0}:\s+{1}'.format( + r'^(?!\#).*{0}:\s+{1}'.format( sdctype, gene_timestamp_regex(brtype, prefix='', nocookie=True)), re.VERBOSE) From 18ff52023c15e5c0f8930e850eaa90c79252b0d0 Mon Sep 17 00:00:00 2001 From: John Lee Date: Sun, 1 Mar 2020 20:21:23 +0000 Subject: [PATCH 34/90] Didn't really intend the leading blank lines here --- orgparse/tests/test_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index 7803bec..7c17f7f 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -93,7 +93,7 @@ def test_iter_node(): def test_commented_headings_do_not_appear_as_children(): - root = loads(""" + root = loads("""\ * H1 #** H2 ** H3 @@ -118,7 +118,7 @@ def test_commented_headings_do_not_appear_as_children(): def test_commented_clock_entries_are_ignored_by_node_clock(): - root = loads(""" + root = loads("""\ * Heading # * Floss # SCHEDULED: <2019-06-22 Sat 08:30 .+1w> @@ -132,7 +132,7 @@ def test_commented_clock_entries_are_ignored_by_node_clock(): def test_commented_scheduled_marker_is_ignored_by_node_scheduled(): - root = loads(""" + root = loads("""\ * Heading # SCHEDULED: <2019-06-22 Sat 08:30 .+1w> """) From 34fd9e2ad8dc938cbdac21c68f7ac21000696a7e Mon Sep 17 00:00:00 2001 From: John Lee Date: Sun, 1 Mar 2020 20:21:44 +0000 Subject: [PATCH 35/90] Test get_property behaviour for comments too Passes this test at least... --- orgparse/tests/test_data.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index 7c17f7f..7c2cc56 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -139,3 +139,15 @@ def test_commented_scheduled_marker_is_ignored_by_node_scheduled(): [node] = root.children[0] assert node.heading == "Heading" assert node.scheduled.start is None + + +def test_commented_property_is_ignored_by_node_get_property(): + root = loads("""\ +* Heading +# :PROPERTIES: +# :PROPER-TEA: backup +# :END: +""") + [node] = root.children[0] + assert node.heading == "Heading" + assert node.get_property("PROPER-TEA") is None From 05bf2a2364ff612a571082c42a9b71011642c92e Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 2 Mar 2020 07:53:00 +0000 Subject: [PATCH 36/90] update authors --- orgparse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index da57422..078291d 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -117,7 +117,7 @@ from .utils.py3compat import basestring __version__ = '0.1.3dev0' -__author__ = 'Takafumi Arakaki' +__author__ = 'Takafumi Arakaki, Dmitrii Gerasimov' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] From b12942024757852c14ede64043532e68ce175ca1 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 2 Mar 2020 07:53:34 +0000 Subject: [PATCH 37/90] bump version --- orgparse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 078291d..4d6a617 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -116,7 +116,7 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.1.3dev0' +__version__ = '0.1.3' __author__ = 'Takafumi Arakaki, Dmitrii Gerasimov' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] From becddb11f9d113f98ac2b45c9348fb0f3d51b4e2 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 2 Mar 2020 07:55:00 +0000 Subject: [PATCH 38/90] bump to 0.1.4dev0 --- orgparse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 4d6a617..3d4e497 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -116,7 +116,7 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.1.3' +__version__ = '0.1.4dev0' __author__ = 'Takafumi Arakaki, Dmitrii Gerasimov' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] From b30638da3f73020c0115d604b6da76945bf35ce7 Mon Sep 17 00:00:00 2001 From: Jonas Schlagenhauf Date: Sun, 19 Apr 2020 13:44:15 +0200 Subject: [PATCH 39/90] Allow to pass a custom OrgEnv instance when loading content In the previous version the OrgEnv instance was created inside the parse_lines() method with the only customizable argument being the filename. In this change the creation of the instance happens in the load/loads/loadi methods and can be optionally passed as an argument from outside. --- orgparse/__init__.py | 12 ++++++------ orgparse/node.py | 8 ++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 3d4e497..6fc4528 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -122,7 +122,7 @@ __all__ = ["load", "loads", "loadi"] -def load(path): +def load(path, env=None): """ Load org-mode document from a file. @@ -139,24 +139,24 @@ def load(path): orgfile = path filename = path.name if hasattr(path, 'name') else '' return loadi((l.rstrip('\n') for l in orgfile.readlines()), - filename=filename) + filename=filename, env=env) -def loads(string, filename=''): +def loads(string, filename='', env=None): """ Load org-mode document from a string. :rtype: :class:`orgparse.node.OrgRootNode` """ - return loadi(string.splitlines(), filename=filename) + return loadi(string.splitlines(), filename=filename, env=env) -def loadi(lines, filename=''): +def loadi(lines, filename='', env=None): """ Load org-mode document from an iterative object. :rtype: :class:`orgparse.node.OrgRootNode` """ - return parse_lines(lines, filename=filename) + return parse_lines(lines, filename=filename, env=env) diff --git a/orgparse/node.py b/orgparse/node.py index a725c38..be61c30 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1256,8 +1256,12 @@ def repeated_tasks(self): return self._repeated_tasks -def parse_lines(lines, filename): - env = OrgEnv(filename=filename) +def parse_lines(lines, filename, env=None): + if not env: + env = OrgEnv(filename=filename) + elif env and env.filename != filename: + raise ValueError('If env is specified, filename must match') + # parse into node of list (environment will be parsed) nodelist = list(env.from_chunks(lines_to_chunks(lines))) # parse headings (level, TODO, TAGs, and heading) From df4afc43a97a7de5c9d6cda764dc2ac549d861dd Mon Sep 17 00:00:00 2001 From: Jonas Schlagenhauf Date: Sun, 19 Apr 2020 16:23:22 +0200 Subject: [PATCH 40/90] Add unit tests for setting custom todo keys --- orgparse/tests/test_misc.py | 51 ++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index 31584a8..61f4802 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -1,11 +1,13 @@ from .. import load, loads +from ..node import OrgEnv + def test_stars(): # https://github.com/karlicoss/orgparse/issues/7#issuecomment-533732660 root = loads(""" * Heading with text (A) -The following line is not a heading, because it begins with a +The following line is not a heading, because it begins with a star but has no spaces afterward, just a newline: * @@ -33,3 +35,50 @@ def test_stars(): assert h1.heading == 'Heading with text (A)' assert h2.heading == '' + +def test_parse_custom_todo_keys(): + todo_keys = ['TODO', 'CUSTOM1', 'ANOTHER_KEYWORD'] + done_keys = ['DONE', 'A'] + filename = '' # default for loads + content = """ +* TODO Heading with a default todo keyword + +* DONE Heading with a default done keyword + +* CUSTOM1 Heading with a custom todo keyword + +* ANOTHER_KEYWORD Heading with a long custom todo keyword + +* A Heading with a short custom done keyword + """ + + env = OrgEnv(todos=todo_keys, dones=done_keys, filename=filename) + root = loads(content, env=env) + + assert root.env.all_todo_keys == ['TODO', 'CUSTOM1', + 'ANOTHER_KEYWORD', 'DONE', 'A'] + assert len(root.children) == 5 + assert root.children[0].todo == 'TODO' + assert root.children[1].todo == 'DONE' + assert root.children[2].todo == 'CUSTOM1' + assert root.children[3].todo == 'ANOTHER_KEYWORD' + assert root.children[4].todo == 'A' + + +def test_add_custom_todo_keys(): + todo_keys = ['CUSTOM_TODO'] + done_keys = ['CUSTOM_DONE'] + filename = '' # default for loads + content = """#+TODO: COMMENT_TODO | COMMENT_DONE + """ + + env = OrgEnv(filename=filename) + env.add_todo_keys(todos=todo_keys, dones=done_keys) + + # check that only the custom keys are know before parsing + assert env.all_todo_keys == ['CUSTOM_TODO', 'CUSTOM_DONE'] + + # after parsing, all keys are set + root = loads(content, filename, env) + assert root.env.all_todo_keys == ['CUSTOM_TODO', 'COMMENT_TODO', + 'CUSTOM_DONE', 'COMMENT_DONE'] From fb12e3591b2766566c069c7c03a2a29bdbd2d60b Mon Sep 17 00:00:00 2001 From: Jonas Schlagenhauf Date: Sun, 19 Apr 2020 16:23:45 +0200 Subject: [PATCH 41/90] Remove unneccesary conditional in argument check --- orgparse/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/node.py b/orgparse/node.py index be61c30..9397ae0 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1259,7 +1259,7 @@ def repeated_tasks(self): def parse_lines(lines, filename, env=None): if not env: env = OrgEnv(filename=filename) - elif env and env.filename != filename: + elif env.filename != filename: raise ValueError('If env is specified, filename must match') # parse into node of list (environment will be parsed) From 64cebedc3830e8329df63d63c429c71926e80f52 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 19 Apr 2020 22:35:14 +0100 Subject: [PATCH 42/90] switch to github actions and CI pypi releases --- .github/workflows/main.yml | 65 +++++++++++++++++++++++ .travis.yml | 5 -- Makefile | 23 --------- README.rst | 5 -- doc/source/conf.py | 1 + orgparse/__init__.py | 6 --- scripts/ci/run | 7 +++ scripts/release | 66 ++++++++++++++++++++++++ setup.py | 103 +++++++++++++++++++++---------------- tox.ini | 2 +- 10 files changed, 198 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml create mode 100755 scripts/ci/run create mode 100755 scripts/release diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..0691ab1 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,65 @@ +# see https://github.com/karlicoss/pymplate for up-to-date reference + +name: CI +on: [push] + +env: + # useful for scripts & sometimes tests to know + CI: true + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + # fuck me. https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#adding-a-system-path + - run: echo "::add-path::$HOME/.local/bin" + + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions/checkout@v2 + with: + submodules: recursive + + # uncomment for SSH debugging + # - uses: mxschmitt/action-tmate@v2 + + - run: scripts/ci/run + + pypi: + runs-on: ubuntu-latest + needs: [build] # add all other jobs here + + steps: + - run: echo "::add-path::$HOME/.local/bin" + + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: 'release to test pypi' + # always deploy merged master to test pypi + if: github.event.ref == 'refs/heads/master' + env: + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }} + run: pip3 install --user wheel twine && scripts/release --test + + - name: 'release to pypi' + # always deploy tags to release pypi + # TODO filter release tags only? + if: startsWith(github.event.ref, 'refs/tags') + env: + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + run: pip3 install --user wheel twine && scripts/release + +# todo generate mypy coverage artifacts? diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4854e53..0000000 --- a/.travis.yml +++ /dev/null @@ -1,5 +0,0 @@ -language: python -before_install: - pip install tox -script: - tox diff --git a/Makefile b/Makefile index 45be100..bc585a1 100644 --- a/Makefile +++ b/Makefile @@ -13,29 +13,6 @@ cog: orgparse/__init__.py orgparse/__init__.py: README.rst cd orgparse && cog.py -r __init__.py -.PHONY: clean -clean: - rm -rf dist/* - - build: clean cog python3 setup.py sdist bdist_wheel -targets := $(wildcard dist/*) - -check: build $(targets) - twine check $(targets) - - - -## https://packaging.python.org/guides/using-testpypi -.PHONY: test-upload -test-upload: check $(targets) - twine upload --verbose --repository-url https://test.pypi.org/legacy/ $(targets) - - -## Upload to PyPI -.PHONY: upload -upload: check $(target) - twine upload --verbose $(targets) - diff --git a/README.rst b/README.rst index 3d29383..130e7ac 100644 --- a/README.rst +++ b/README.rst @@ -6,11 +6,6 @@ * `Documentation (Read the Docs) `_ * `Repository (at GitHub) `_ * `PyPI `_ -* `Travis CI `_ |build-status| - -.. |build-status| - image:: https://travis-ci.org/karlicoss/orgparse.svg?branch=master - :target: https://travis-ci.org/karlicoss/orgparse Install ------- diff --git a/doc/source/conf.py b/doc/source/conf.py index 222f2d5..3f90daa 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,6 +23,7 @@ copyright = u'2012, Takafumi Arakaki' # The short X.Y version. +# TODO use setup.py for version version = orgparse.__version__ # The full version, including alpha/beta/rc tags. release = orgparse.__version__ diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 6fc4528..24ebc50 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -12,11 +12,6 @@ * `Documentation (Read the Docs) `_ * `Repository (at GitHub) `_ * `PyPI `_ -* `Travis CI `_ |build-status| - -.. |build-status| - image:: https://travis-ci.org/karlicoss/orgparse.svg?branch=master - :target: https://travis-ci.org/karlicoss/orgparse Install ------- @@ -116,7 +111,6 @@ from .node import parse_lines from .utils.py3compat import basestring -__version__ = '0.1.4dev0' __author__ = 'Takafumi Arakaki, Dmitrii Gerasimov' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] diff --git a/scripts/ci/run b/scripts/ci/run new file mode 100755 index 0000000..c49e635 --- /dev/null +++ b/scripts/ci/run @@ -0,0 +1,7 @@ +#!/bin/bash -eu + +cd "$(dirname "$0")" +cd ../.. + +pip3 install --user tox +tox diff --git a/scripts/release b/scripts/release new file mode 100755 index 0000000..0ec687f --- /dev/null +++ b/scripts/release @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +''' +Run [[file:scripts/release][scripts/release]] to deploy Python package onto [[https://pypi.org][PyPi]] and [[https://test.pypi.org][test PyPi]]. + +The script expects =TWINE_PASSWORD= environment variable to contain the [[https://pypi.org/help/#apitoken][PyPi token]] (not the password!). + +The script can be run manually. +It's also running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]]. Packages are deployed on: +- every master commit, onto test pypi +- every new tag, onto production pypi + +You'll need to set =TWINE_PASSWORD= and =TWINE_PASSWORD_TEST= in [[https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets#creating-encrypted-secrets][secrets]] +for Github Actions deployment to work. +''' + +import os +import sys +from pathlib import Path +from subprocess import check_call +import shutil + +is_ci = os.environ.get('CI') is not None + +def main(): + import argparse + p = argparse.ArgumentParser() + p.add_argument('--test', action='store_true', help='use test pypi') + args = p.parse_args() + + extra = [] + if args.test: + extra.extend(['--repository-url', 'https://test.pypi.org/legacy/']) + + root = Path(__file__).absolute().parent.parent + os.chdir(root) # just in case + + if is_ci: + # see https://github.com/actions/checkout/issues/217 + check_call('git fetch --prune --unshallow'.split()) + + dist = root / 'dist' + if dist.exists(): + shutil.rmtree(dist) + + check_call('python3 setup.py sdist bdist_wheel', shell=True) + + TP = 'TWINE_PASSWORD' + password = os.environ.get(TP) + if password is None: + print(f"WARNING: no {TP} passed", file=sys.stderr) + import pip_secrets + password = pip_secrets.token_test if args.test else pip_secrets.token # meh + + check_call([ + 'python3', '-m', 'twine', + 'upload', *dist.iterdir(), + *extra, + ], env={ + 'TWINE_USERNAME': '__token__', + TP: password, + **os.environ, + }) + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index b943662..caf26e7 100644 --- a/setup.py +++ b/setup.py @@ -1,45 +1,58 @@ -import setuptools -from distutils.core import setup - -import orgparse - -setup( - name='orgparse', - version=orgparse.__version__, - packages=[ - 'orgparse', - 'orgparse.utils', - 'orgparse.tests', - 'orgparse.tests.data', - ], - package_data={ - 'orgparse.tests.data': ['*.org'], - }, - - author=orgparse.__author__, - author_email='aka.tkf@gmail.com', - maintainer='Dima Gerasimov (@karlicoss)', - maintainer_email='karlicoss@gmail.com', - - url='https://github.com/karlicoss/orgparse', - license=orgparse.__license__, - - description='orgparse - Emacs org-mode parser in Python', - long_description=orgparse.__doc__, - - keywords='org org-mode emacs', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Topic :: Text Processing :: Markup', - # see: http://pypi.python.org/pypi?%3Aaction=list_classifiers - ], -) +# see https://github.com/karlicoss/pymplate for up-to-date reference +# +from setuptools import setup, find_packages # type: ignore + + +def main(): + import orgparse + setup( + name='orgparse', + use_scm_version={ + 'version_scheme': 'python-simplified-semver', + 'local_scheme': 'dirty-tag', + }, + setup_requires=['setuptools_scm'], + + zip_safe=False, + + packages=[ + 'orgparse', + 'orgparse.utils', + 'orgparse.tests', + 'orgparse.tests.data', + ], + package_data={ + 'orgparse.tests.data': ['*.org'], + }, + + author=orgparse.__author__, + author_email='aka.tkf@gmail.com', + maintainer='Dima Gerasimov (@karlicoss)', + maintainer_email='karlicoss@gmail.com', + + url='https://github.com/karlicoss/orgparse', + license=orgparse.__license__, + + description='orgparse - Emacs org-mode parser in Python', + long_description=orgparse.__doc__, + + keywords='org org-mode emacs', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Text Processing :: Markup', + # see: http://pypi.python.org/pypi?%3Aaction=list_classifiers + ], + ) + + +if __name__ == '__main__': + main() diff --git a/tox.ini b/tox.ini index a1cc1a1..4b98d01 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py36 +envlist = py2,py3 [testenv] deps = pytest From 9184c9a78053962de14057e69fc9221c74ba003f Mon Sep 17 00:00:00 2001 From: Martin Michlmayr Date: Sat, 11 Jul 2020 09:39:08 +0800 Subject: [PATCH 43/90] Fix typos in docstrings --- orgparse/__init__.py | 2 +- orgparse/date.py | 2 +- orgparse/node.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 24ebc50..1e9b4d8 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -121,7 +121,7 @@ def load(path, env=None): Load org-mode document from a file. :type path: str or file-like - :arg path: Path to org file or file-like object of a org document. + :arg path: Path to org file or file-like object of an org document. :rtype: :class:`orgparse.node.OrgRootNode` diff --git a/orgparse/date.py b/orgparse/date.py index 1cce7bf..dc6e89f 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -15,7 +15,7 @@ def total_minutes(td): def gene_timestamp_regex(brtype, prefix=None, nocookie=False): """ - Generate timetamp regex for active/inactive/nobrace brace type + Generate timestamp regex for active/inactive/nobrace brace type :type brtype: {'active', 'inactive', 'nobrace'} :arg brtype: diff --git a/orgparse/node.py b/orgparse/node.py index 9397ae0..da26f0d 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -97,7 +97,7 @@ def parse_heading_todos(heading, todo_candidates): def parse_heading_priority(heading): """ - Get priority and heading without priority field.. + Get priority and heading without priority field. >>> parse_heading_priority('HEADING') ('HEADING', None) @@ -347,7 +347,7 @@ class OrgBaseNode(Sequence): def __init__(self, env, index=None): """ - Create a :class:`OrgBaseNode` object. + Create an :class:`OrgBaseNode` object. :type env: :class:`OrgEnv` :arg env: This will be set to the :attr:`env` attribute. @@ -475,9 +475,9 @@ def get_parent(self, max_level=None): of the ancestor node to return. For example, ``get_parent(max_level=0)`` returns a root node. - In general case, it specify a maximum level of the + In the general case, it specify a maximum level of the desired ancestor node. If there is no ancestor node - which level is equal to ``max_level``, this function + whose level is equal to ``max_level``, this function try to find an ancestor node which level is smaller than ``max_level``. @@ -587,7 +587,7 @@ def children(self): >>> c2 is n4 True - Note the difference to ``n1[1:]``, which returns the Node 3 also.: + Note the difference to ``n1[1:]``, which returns the Node 3 also: >>> (m1, m2, m3) = list(n1[1:]) >>> m2 is n3 @@ -664,7 +664,7 @@ def _get_tags(self, inher=False): @property def tags(self): """ - Tag of this and parents node. + Tags of this and parent's node. >>> from orgparse import loads >>> n2 = loads(''' @@ -1212,7 +1212,7 @@ def has_date(self): @property def repeated_tasks(self): """ - Get repeated tasks marked DONE in a entry having repeater. + Get repeated tasks marked DONE in an entry having repeater. :rtype: list of :class:`orgparse.date.OrgDateRepeatedTask` From 292a152b0ff20c45b4b9f91fb186121a9209ef9c Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 1 Nov 2020 13:36:46 +0000 Subject: [PATCH 44/90] update CI scripts, add mypy checks --- {scripts => .ci}/release | 0 .ci/run | 24 +++++ .github/workflows/main.yml | 37 ++++--- .gitignore | 196 ++++++++++++++++++++++++++++++++++++- Makefile | 4 - mypy.ini | 10 ++ scripts/ci/run | 7 -- setup.py | 20 ++-- tox.ini | 22 ++++- 9 files changed, 283 insertions(+), 37 deletions(-) rename {scripts => .ci}/release (100%) create mode 100755 .ci/run create mode 100644 mypy.ini delete mode 100755 scripts/ci/run diff --git a/scripts/release b/.ci/release similarity index 100% rename from scripts/release rename to .ci/release diff --git a/.ci/run b/.ci/run new file mode 100755 index 0000000..a0a314a --- /dev/null +++ b/.ci/run @@ -0,0 +1,24 @@ +#!/bin/bash -eu + +cd "$(dirname "$0")" +cd .. + +if ! command -v sudo; then + # CI or Docker sometimes don't have it, so useful to have a dummy + function sudo { + "$@" + } +fi + +if ! [ -z "$CI" ]; then + # install OS specific stuff here + if [[ "$OSTYPE" == "darwin"* ]]; then + # macos + : + else + : + fi +fi + +pip3 install --user tox +tox diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0691ab1..539d13c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,7 +1,14 @@ # see https://github.com/karlicoss/pymplate for up-to-date reference name: CI -on: [push] +on: + push: + branches: '*' + tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi + # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug: + # P.S. fuck made up yaml DSLs. + # TODO cron? + workflow_dispatch: # needed to trigger workflows manually env: # useful for scripts & sometimes tests to know @@ -9,15 +16,16 @@ env: jobs: build: - runs-on: ubuntu-latest - strategy: matrix: + platform: [ubuntu-latest] # macos-latest] # TODO windows-latest?? python-version: [3.5, 3.6, 3.7, 3.8] + runs-on: ${{ matrix.platform }} + steps: - # fuck me. https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#adding-a-system-path - - run: echo "::add-path::$HOME/.local/bin" + # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - uses: actions/setup-python@v1 with: @@ -30,14 +38,21 @@ jobs: # uncomment for SSH debugging # - uses: mxschmitt/action-tmate@v2 - - run: scripts/ci/run + - run: .ci/run + + - uses: actions/upload-artifact@v2 + with: + name: .mypy-coverage_${{ matrix.platform }}_${{ matrix.python-version }} + path: .mypy-coverage/ + pypi: runs-on: ubuntu-latest needs: [build] # add all other jobs here steps: - - run: echo "::add-path::$HOME/.local/bin" + # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - uses: actions/setup-python@v1 with: @@ -52,14 +67,12 @@ jobs: if: github.event.ref == 'refs/heads/master' env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }} - run: pip3 install --user wheel twine && scripts/release --test + run: pip3 install --user wheel twine && .ci/release --test - name: 'release to pypi' # always deploy tags to release pypi - # TODO filter release tags only? + # NOTE: release tags are guarded by on: push: tags on the top if: startsWith(github.event.ref, 'refs/tags') env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - run: pip3 install --user wheel twine && scripts/release - -# todo generate mypy coverage artifacts? + run: pip3 install --user wheel twine && .ci/release diff --git a/.gitignore b/.gitignore index 0fb9476..a06322a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,195 @@ -.tox + +# Created by https://www.toptal.com/developers/gitignore/api/python,emacs +# Edit at https://www.toptal.com/developers/gitignore?templates=python,emacs + +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg MANIFEST -doc/build + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# profiling data +.prof + +# End of https://www.toptal.com/developers/gitignore/api/python,emacs diff --git a/Makefile b/Makefile index bc585a1..c22f6de 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,3 @@ doc: cog cog: orgparse/__init__.py orgparse/__init__.py: README.rst cd orgparse && cog.py -r __init__.py - -build: clean cog - python3 setup.py sdist bdist_wheel - diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..0f8386f --- /dev/null +++ b/mypy.ini @@ -0,0 +1,10 @@ +[mypy] +pretty = True +show_error_context = True +show_error_codes = True +check_untyped_defs = True +namespace_packages = True + +# an example of suppressing +# [mypy-my.config.repos.pdfannots.pdfannots] +# ignore_errors = True diff --git a/scripts/ci/run b/scripts/ci/run deleted file mode 100755 index c49e635..0000000 --- a/scripts/ci/run +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -eu - -cd "$(dirname "$0")" -cd ../.. - -pip3 install --user tox -tox diff --git a/setup.py b/setup.py index caf26e7..dfc1292 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,15 @@ # see https://github.com/karlicoss/pymplate for up-to-date reference # -from setuptools import setup, find_packages # type: ignore +from setuptools import setup, find_namespace_packages # type: ignore def main(): + pkg = 'orgparse' + subpkgs = find_namespace_packages('.', include=(pkg + '.*',)) + import orgparse setup( - name='orgparse', + name=pkg, use_scm_version={ 'version_scheme': 'python-simplified-semver', 'local_scheme': 'dirty-tag', @@ -15,13 +18,9 @@ def main(): zip_safe=False, - packages=[ - 'orgparse', - 'orgparse.utils', - 'orgparse.tests', - 'orgparse.tests.data', - ], + packages=[pkg, *subpkgs], package_data={ + 'pkg': ['py.typed'], # todo need the rest as well?? 'orgparse.tests.data': ['*.org'], }, @@ -51,6 +50,11 @@ def main(): 'Topic :: Text Processing :: Markup', # see: http://pypi.python.org/pypi?%3Aaction=list_classifiers ], + + extras_require={ + 'testing': ['pytest'], + 'linting': ['pytest', 'mypy', 'lxml'], # lxml for mypy coverage report + }, ) diff --git a/tox.ini b/tox.ini index 4b98d01..24fd480 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,20 @@ [tox] -envlist = py2,py3 +minversion = 3.5 +# relies on the correct version of Python installed +envlist = py2,py3,mypy + [testenv] -deps = - pytest -commands = pytest --doctest-modules --ignore-glob='**/_py3compat.py' orgparse +commands = + pip install -e .[testing] + python -m pytest -rap --doctest-modules --ignore-glob='**/_py3compat.py' orgparse {posargs} + + +[testenv:mypy] +skip_install = true +commands = + pip install -e .[linting] + python -m mypy orgparse \ + # txt report is a bit more convenient to view on CI + --txt-report .mypy-coverage \ + --html-report .mypy-coverage \ + {posargs} From c465163e98619ff534ba5d99ff0fdbb8e2d7a386 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 1 Nov 2020 14:27:13 +0000 Subject: [PATCH 45/90] ci: add mypy initial annotations --- orgparse/__init__.py | 7 +-- orgparse/date.py | 28 ++++++----- orgparse/node.py | 68 ++++++++++++++------------- orgparse/py.typed | 0 orgparse/tests/data/00_simple.py | 26 +++++----- orgparse/tests/data/01_attributes.py | 10 ++-- orgparse/tests/data/02_tree_struct.py | 2 +- orgparse/tests/test_data.py | 2 +- orgparse/tests/test_hugedata.py | 4 +- orgparse/utils/py3compat.py | 10 ++-- 10 files changed, 83 insertions(+), 74 deletions(-) create mode 100644 orgparse/py.typed diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 1e9b4d8..7930945 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -107,8 +107,9 @@ # [[[end]]] import codecs +from typing import Iterable -from .node import parse_lines +from .node import parse_lines, OrgNode # todo basenode?? from .utils.py3compat import basestring __author__ = 'Takafumi Arakaki, Dmitrii Gerasimov' @@ -136,7 +137,7 @@ def load(path, env=None): filename=filename, env=env) -def loads(string, filename='', env=None): +def loads(string: str, filename='', env=None) -> OrgNode: """ Load org-mode document from a string. @@ -146,7 +147,7 @@ def loads(string, filename='', env=None): return loadi(string.splitlines(), filename=filename, env=env) -def loadi(lines, filename='', env=None): +def loadi(lines: Iterable[str], filename='', env=None) -> OrgNode: """ Load org-mode document from an iterative object. diff --git a/orgparse/date.py b/orgparse/date.py index dc6e89f..9b1d8fa 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -1,5 +1,8 @@ import datetime import re +from typing import Union, Tuple, Optional, List + +DateIsh = Union[datetime.date, datetime.datetime] def total_seconds(td): @@ -191,7 +194,7 @@ def __init__(self, start, end=None, active=None): self._active = self._active_default if active is None else active @staticmethod - def _to_date(date): + def _to_date(date) -> DateIsh: if isinstance(date, (tuple, list)): if len(date) == 3: return datetime.date(*date) @@ -270,15 +273,15 @@ def end(self): """ return self._end - def is_active(self): + def is_active(self) -> bool: """Return true if the date is active""" return self._active - def has_end(self): + def has_end(self) -> bool: """Return true if it has the end date""" return bool(self._end) - def has_time(self): + def has_time(self) -> bool: """ Return true if the start date has time field @@ -290,7 +293,7 @@ def has_time(self): """ return isinstance(self._start, datetime.datetime) - def has_overlap(self, other): + def has_overlap(self, other) -> bool: """ Test if it has overlap with other :class:`OrgDate` instance @@ -336,10 +339,11 @@ def _as_datetime(date): return date @staticmethod - def _daterange_from_groupdict(dct, prefix=''): + def _daterange_from_groupdict(dct, prefix='') -> Tuple[List, Optional[List]]: start_keys = ['year', 'month', 'day', 'hour' , 'min'] end_keys = ['year', 'month', 'day', 'end_hour', 'end_min'] start_range = list(map(int, filter(None, (dct[prefix + k] for k in start_keys)))) + end_range: Optional[List] end_range = list(map(int, filter(None, (dct[prefix + k] for k in end_keys)))) if len(end_range) < len(end_keys): end_range = None @@ -350,7 +354,7 @@ def _datetuple_from_groupdict(cls, dct, prefix=''): return cls._daterange_from_groupdict(dct, prefix=prefix)[0] @classmethod - def list_from_str(cls, string): + def list_from_str(cls, string: str) -> List['OrgDate']: """ Parse string and return a list of :class:`OrgDate` objects @@ -434,7 +438,9 @@ class OrgDateSDCBase(OrgDate): # FIXME: use OrgDate.from_str @classmethod def from_str(cls, string): - match = cls._re.search(string) + rgx = cls._re + assert rgx is not None + match = rgx.search(string) if match: mdict = match.groupdict() start = cls._datetuple_from_groupdict(mdict) @@ -529,7 +535,7 @@ def is_duration_consistent(self): self._duration == total_minutes(self.duration)) @classmethod - def from_str(cls, line): + def from_str(cls, line: str) -> 'OrgDateClock': """ Get CLOCK from given string. @@ -545,8 +551,8 @@ def from_str(cls, line): ymdhm2 = groups[5:10] hm3 = groups[10:] return cls( - datetime.datetime(*ymdhm1), - datetime.datetime(*ymdhm2), + datetime.datetime(*ymdhm1), # type: ignore[arg-type] + datetime.datetime(*ymdhm2), # type: ignore[arg-type] hm3[0] * 60 + hm3[1], ) diff --git a/orgparse/node.py b/orgparse/node.py index da26f0d..5be993c 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1,5 +1,6 @@ import re import itertools +from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict try: from collections.abc import Sequence except ImportError: @@ -10,8 +11,8 @@ from .utils.py3compat import PY3, unicode -def lines_to_chunks(lines): - chunk = [] +def lines_to_chunks(lines: Iterable[str]) -> Iterable[List[str]]: + chunk: List[str] = [] for l in lines: if RE_NODE_HEADER.search(l): yield chunk @@ -42,7 +43,7 @@ def parse_heading_level(heading): RE_HEADING_STARS = re.compile(r'^(\*+)\s+(.*?)\s*$') -def parse_heading_tags(heading): +def parse_heading_tags(heading: str) -> Tuple[str, List[str]]: """ Get first tags and heading without tags @@ -117,8 +118,8 @@ def parse_heading_priority(heading): RE_HEADING_PRIORITY = re.compile(r'^\s*\[#([A-Z0-9])\] ?(.*)$') - -def parse_property(line): +PropertyValue = Union[str, int] +def parse_property(line: str) -> Tuple[Optional[str], Optional[PropertyValue]]: """ Get property from given string. @@ -129,7 +130,7 @@ def parse_property(line): """ prop_key = None - prop_val = None + prop_val: Optional[Union[str, int]] = None match = RE_PROP.search(line) if match: prop_key = match.group(1) @@ -625,7 +626,7 @@ def from_chunk(cls, env, lines): return self def _parse_comments(self): - special_comments = {} + special_comments: Dict[str, List[str]] = {} for line in self._lines: parsed = parse_comment(line) if parsed: @@ -751,21 +752,22 @@ class OrgNode(OrgBaseNode): """ - def __init__(self, *args, **kwds): + def __init__(self, *args, **kwds) -> None: super(OrgNode, self).__init__(*args, **kwds) - self._heading = None + # fixme instead of casts, should organize code in such a way that they aren't necessary + self._heading = cast(str, None) self._level = None - self._tags = None + self._tags = cast(List[str], None) self._todo = None self._priority = None - self._properties = {} + self._properties: Dict[str, PropertyValue] = {} self._scheduled = OrgDate(None) self._deadline = OrgDate(None) self._closed = OrgDate(None) - self._timestamps = [] - self._clocklist = [] - self._body_lines = [] - self._repeated_tasks = [] + self._timestamps: List[OrgDate] = [] + self._clocklist: List[OrgDateClock] = [] + self._body_lines: List[str] = [] + self._repeated_tasks: List[OrgDateRepeatedTask] = [] # parser @@ -773,7 +775,7 @@ def _parse_pre(self): """Call parsers which must be called before tree structuring""" self._parse_heading() # FIXME: make the following parsers "lazy" - ilines = iter(self._lines) + ilines: Iterator[str] = iter(self._lines) try: next(ilines) # skip heading except StopIteration: @@ -785,7 +787,7 @@ def _parse_pre(self): ilines = self._iparse_timestamps(ilines) self._body_lines = list(ilines) - def _parse_heading(self): + def _parse_heading(self) -> None: heading = self._lines[0] (heading, self._level) = parse_heading_level(heading) (heading, self._tags) = parse_heading_tags(heading) @@ -800,7 +802,7 @@ def _parse_heading(self): # If the item returned by the input iterator must be dedicated to # the parser, do not yield the item or yield it as-is otherwise. - def _iparse_sdc(self, ilines): + def _iparse_sdc(self, ilines: Iterator[str]) -> Iterator[str]: """ Parse SCHEDULED, DEADLINE and CLOSED time tamps. @@ -821,24 +823,24 @@ def _iparse_sdc(self, ilines): for line in ilines: yield line - def _iparse_clock(self, ilines): - self._clocklist = clocklist = [] + def _iparse_clock(self, ilines: Iterator[str]) -> Iterator[str]: + self._clocklist = [] for line in ilines: cl = OrgDateClock.from_str(line) if cl: - clocklist.append(cl) + self._clocklist.append(cl) else: yield line - def _iparse_timestamps(self, ilines): - self._timestamps = timestamps = [] - timestamps.extend(OrgDate.list_from_str(self._heading)) + def _iparse_timestamps(self, ilines: Iterator[str]) -> Iterator[str]: + self._timestamps = [] + self._timestamps.extend(OrgDate.list_from_str(self._heading)) for l in ilines: - timestamps.extend(OrgDate.list_from_str(l)) + self._timestamps.extend(OrgDate.list_from_str(l)) yield l - def _iparse_properties(self, ilines): - self._properties = properties = {} + def _iparse_properties(self, ilines: Iterator[str]) -> Iterator[str]: + self._properties = {} in_property_field = False for line in ilines: if in_property_field: @@ -846,8 +848,8 @@ def _iparse_properties(self, ilines): break else: (key, val) = parse_property(line) - if key: - properties.update({key: val}) + if key is not None and val is not None: + self._properties.update({key: val}) elif line.find(":PROPERTIES:") >= 0: in_property_field = True else: @@ -855,8 +857,8 @@ def _iparse_properties(self, ilines): for line in ilines: yield line - def _iparse_repeated_tasks(self, ilines): - self._repeated_tasks = repeated_tasks = [] + def _iparse_repeated_tasks(self, ilines: Iterator[str]) -> Iterator[str]: + self._repeated_tasks = [] for line in ilines: match = self._repeated_tasks_re.search(line) if match: @@ -865,7 +867,7 @@ def _iparse_repeated_tasks(self, ilines): done_state = mdict['done'] todo_state = mdict['todo'] date = OrgDate.from_str(mdict['date']) - repeated_tasks.append( + self._repeated_tasks.append( OrgDateRepeatedTask(date.start, todo_state, done_state)) else: yield line @@ -1256,7 +1258,7 @@ def repeated_tasks(self): return self._repeated_tasks -def parse_lines(lines, filename, env=None): +def parse_lines(lines: Iterable[str], filename, env=None) -> OrgNode: if not env: env = OrgEnv(filename=filename) elif env.filename != filename: diff --git a/orgparse/py.typed b/orgparse/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/orgparse/tests/data/00_simple.py b/orgparse/tests/data/00_simple.py index 8f7da41..0e68f13 100644 --- a/orgparse/tests/data/00_simple.py +++ b/orgparse/tests/data/00_simple.py @@ -13,18 +13,18 @@ def tags(nums): data = [ - nodedict(i, *vals) for (i, vals) in enumerate([ - [1, 'TODO1', tags([1]), tags(range(1, 2))], - [2, 'TODO2', tags([2]), tags(range(1, 3))], - [3, 'TODO3', tags([3]), tags(range(1, 4))], - [4, 'TODO4', tags([4]), tags(range(1, 5))], - [2, None, tags([]), tags([1])], - [2, None, tags([]), tags([1])], - [1, None, tags([2]), tags([2])], - [2, None, tags([2]), tags([2])], - [3, None, tags([]), tags([2])], - [5, None, tags([3, 4]), tags([2, 3, 4])], - [4, None, tags([1]), tags([1, 2])], - [2, None, tags([]), tags([2])], + nodedict(i, *vals) for (i, vals) in enumerate([ # type: ignore[misc] + [1, 'TODO1', tags([1]) , tags(range(1, 2))], + [2, 'TODO2', tags([2]) , tags(range(1, 3))], + [3, 'TODO3', tags([3]) , tags(range(1, 4))], + [4, 'TODO4', tags([4]) , tags(range(1, 5))], + [2, None , tags([]) , tags([1]) ], + [2, None , tags([]) , tags([1]) ], + [1, None , tags([2]) , tags([2]) ], + [2, None , tags([2]) , tags([2]) ], + [3, None , tags([]) , tags([2]) ], + [5, None , tags([3, 4]), tags([2, 3, 4]) ], + [4, None , tags([1]) , tags([1, 2]) ], + [2, None , tags([]) , tags([2]) ], [1], ])] diff --git a/orgparse/tests/data/01_attributes.py b/orgparse/tests/data/01_attributes.py index 8359c26..1ced627 100644 --- a/orgparse/tests/data/01_attributes.py +++ b/orgparse/tests/data/01_attributes.py @@ -1,9 +1,13 @@ +from typing import Dict, Any + from orgparse.date import ( OrgDate, OrgDateScheduled, OrgDateDeadline, OrgDateClosed, OrgDateClock, ) -node1 = dict( +Raw = Dict[str, Any] + +node1: Raw = dict( heading="A node with a lot of attributes", priority='A', scheduled=OrgDateScheduled((2010, 8, 6)), @@ -27,7 +31,7 @@ - <2019-08-10 Sat 16:30-17:30> TimeRange""" ) -node2 = dict( +node2: Raw = dict( heading="A node without any attributed", priority=None, scheduled=OrgDate(None), @@ -40,7 +44,7 @@ body="", ) -node3 = dict( +node3: Raw = dict( heading="range in deadline", priority=None, scheduled=OrgDate(None), diff --git a/orgparse/tests/data/02_tree_struct.py b/orgparse/tests/data/02_tree_struct.py index ae65fad..c93aa56 100644 --- a/orgparse/tests/data/02_tree_struct.py +++ b/orgparse/tests/data/02_tree_struct.py @@ -5,7 +5,7 @@ def nodedict(parent, children=[], previous=None, next=None): next_same_level_heading=next) -data = [nodedict(*args) for args in [ +data = [nodedict(*args) for args in [ # type: ignore[misc] # G0 (None, [], None, 'G1-H1'), # G1 diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index 7c2cc56..fc2e2c2 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -12,7 +12,7 @@ def load_data(path): """Load data from python file""" - ns = {} + ns = {} # type: ignore execfile(path, ns) return ns['data'] diff --git a/orgparse/tests/test_hugedata.py b/orgparse/tests/test_hugedata.py index 171d5b2..b768ae8 100644 --- a/orgparse/tests/test_hugedata.py +++ b/orgparse/tests/test_hugedata.py @@ -1,7 +1,7 @@ try: - import cPickle as pickle + import cPickle as pickle # type: ignore[import] except ImportError: - import pickle + import pickle # type: ignore[no-redef] from .. import loadi diff --git a/orgparse/utils/py3compat.py b/orgparse/utils/py3compat.py index 6a9059e..2d5fa6a 100644 --- a/orgparse/utils/py3compat.py +++ b/orgparse/utils/py3compat.py @@ -2,15 +2,11 @@ PY3 = (sys.version_info[0] >= 3) -try: - # Python 2 +if PY3: + basestring = unicode = str +else: unicode = unicode basestring = basestring -except NameError: - # Python 3 - basestring = unicode = str - -PY3 = (sys.version_info[0] >= 3) if PY3: from ._py3compat import execfile From be5dbf0ae87452d46b95ecfa90bae740820e27b4 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 1 Nov 2020 15:46:31 +0000 Subject: [PATCH 46/90] drop py2/py3.5 support I'd prefer not too, but IMO typing module/mypy annotations are too valuable --- .github/workflows/main.yml | 2 +- setup.py | 8 -------- tox.ini | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 539d13c..eef4b29 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest] # macos-latest] # TODO windows-latest?? - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8] runs-on: ${{ matrix.platform }} diff --git a/setup.py b/setup.py index dfc1292..f3a6a51 100644 --- a/setup.py +++ b/setup.py @@ -39,14 +39,6 @@ def main(): classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Topic :: Text Processing :: Markup', # see: http://pypi.python.org/pypi?%3Aaction=list_classifiers ], diff --git a/tox.ini b/tox.ini index 24fd480..1a07c67 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 3.5 # relies on the correct version of Python installed -envlist = py2,py3,mypy +envlist = py3,mypy [testenv] commands = From 18a836b34e9304c9dff7ddf6c7e3c9311d74748b Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 1 Nov 2020 17:06:01 +0000 Subject: [PATCH 47/90] keep linenumber information in the nodes --- orgparse/node.py | 21 +++++++++++++++++---- orgparse/tests/test_data.py | 4 ++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index 5be993c..738b8ba 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -327,7 +327,7 @@ class OrgBaseNode(Sequence): *** Heading 3 * Heading 4 - It also support sequence protocol. + It also supports sequence protocol. >>> print(root[1]) * Heading 1 @@ -344,9 +344,13 @@ class OrgBaseNode(Sequence): ** Heading 2 *** Heading 3 + Nodes remember the line number information (1-indexed): + + >>> print(root.children[1].linenumber) + 5 """ - def __init__(self, env, index=None): + def __init__(self, env, index=None) -> None: """ Create an :class:`OrgBaseNode` object. @@ -356,8 +360,10 @@ def __init__(self, env, index=None): """ self.env = env + self.linenumber = cast(int, None) # set in parse_lines + # content - self._lines = [] + self._lines: List[str] = [] # FIXME: use `index` argument to set index. (Currently it is # done externally in `parse_lines`.) @@ -1265,7 +1271,14 @@ def parse_lines(lines: Iterable[str], filename, env=None) -> OrgNode: raise ValueError('If env is specified, filename must match') # parse into node of list (environment will be parsed) - nodelist = list(env.from_chunks(lines_to_chunks(lines))) + ch1, ch2 = itertools.tee(lines_to_chunks(lines)) + linenos = itertools.accumulate(itertools.chain([0], (len(c) for c in ch1))) + nodes = env.from_chunks(ch2) + nodelist = [] + for lineno, node in zip(linenos, nodes): + lineno += 1 # in text editors lines are 1-indexed + node.linenumber = lineno + nodelist.append(node) # parse headings (level, TODO, TAGs, and heading) nodelist[0]._index = 0 for (i, node) in enumerate(nodelist[1:], 1): # nodes except root node diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index fc2e2c2..5851513 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -101,20 +101,24 @@ def test_commented_headings_do_not_appear_as_children(): #** H5 * H6 """) + assert root.linenumber == 1 top_level = root.children assert len(top_level) == 2 h1 = top_level[0] assert h1.heading == "H1" assert h1.get_body() == "#** H2" + assert h1.linenumber == 1 [h3] = h1.children assert h3.heading == "H3" assert h3.get_body() == "#* H4\n#** H5" + assert h3.linenumber == 3 h6 = top_level[1] assert h6.heading == "H6" assert len(h6.children) == 0 + assert h6.linenumber == 6 def test_commented_clock_entries_are_ignored_by_node_clock(): From 362f0865b2a281ad3afc761835bf966ef69ba29a Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 1 Nov 2020 17:38:33 +0000 Subject: [PATCH 48/90] fix for parsing empty heading --- orgparse/node.py | 23 ++++++++++++----------- orgparse/tests/test_misc.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index 738b8ba..f527f1a 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1,6 +1,6 @@ import re import itertools -from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict +from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict, Set try: from collections.abc import Sequence except ImportError: @@ -78,7 +78,7 @@ def parse_heading_tags(heading: str) -> Tuple[str, List[str]]: RE_HEADING_TAGS = re.compile(r'(.*?)\s*:([\w@:]+):\s*$') -def parse_heading_todos(heading, todo_candidates): +def parse_heading_todos(heading: str, todo_candidates: List[str]) -> Tuple[str, Optional[str]]: """ Get TODO keyword and heading without TODO keyword. @@ -90,9 +90,10 @@ def parse_heading_todos(heading, todo_candidates): """ for todo in todo_candidates: - todows = '{0} '.format(todo) - if heading.startswith(todows): - return (heading[len(todows):], todo) + if heading == todo: + return ('', todo) + if heading.startswith(todo + ' '): + return (heading[len(todo) + 1:], todo) return (heading, None) @@ -656,7 +657,7 @@ def level(self): """ raise NotImplemented - def _get_tags(self, inher=False): + def _get_tags(self, inher=False) -> Set[str]: """ Return tags @@ -669,7 +670,7 @@ def _get_tags(self, inher=False): return set() @property - def tags(self): + def tags(self) -> Set[str]: """ Tags of this and parent's node. @@ -685,7 +686,7 @@ def tags(self): return self._get_tags(inher=True) @property - def shallow_tags(self): + def shallow_tags(self) -> Set[str]: """ Tags defined for this node (don't look-up parent nodes). @@ -764,7 +765,7 @@ def __init__(self, *args, **kwds) -> None: self._heading = cast(str, None) self._level = None self._tags = cast(List[str], None) - self._todo = None + self._todo: Optional[str] = None self._priority = None self._properties: Dict[str, PropertyValue] = {} self._scheduled = OrgDate(None) @@ -978,7 +979,7 @@ def priority(self): """ return self._priority - def _get_tags(self, inher=False): + def _get_tags(self, inher=False) -> Set[str]: tags = set(self._tags) if inher: parent = self.get_parent() @@ -987,7 +988,7 @@ def _get_tags(self, inher=False): return tags @property - def todo(self): + def todo(self) -> Optional[str]: """ A TODO keyword of this node if exists or None otherwise. diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index 61f4802..1f43e79 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -2,6 +2,18 @@ from ..node import OrgEnv +def test_empty_heading() -> None: + root = loads(''' +* TODO :sometag: + has no heading but still a todo? + it's a bit unclear, but seems to be highligted by emacs.. +''') + [h] = root.children + assert h.todo == 'TODO' + assert h.heading == '' + assert h.tags == {'sometag'} + + def test_stars(): # https://github.com/karlicoss/orgparse/issues/7#issuecomment-533732660 root = loads(""" From c7b719d97bd781fac5d80379a31302e7d191e1d1 Mon Sep 17 00:00:00 2001 From: Samuel MEYNARD Date: Sat, 19 Sep 2020 09:45:06 +0200 Subject: [PATCH 49/90] feat: add get_file_property and get_only_file_property --- orgparse/node.py | 25 +++++++++++++++++++++++-- orgparse/tests/test_misc.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index f527f1a..599120f 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -154,8 +154,8 @@ def parse_comment(line): >>> parse_comment('# not a special comment') # None """ - if line.startswith('#+'): - comment = line.lstrip('#+').split(':', 1) + if re.match(r'\s*#\+', line): + comment = re.split(r':\s+', re.split(r'\s*#\+',line)[1]) if len(comment) == 2: return (comment[0], comment[1].strip()) @@ -725,6 +725,27 @@ def __unicode__(self): def __str__(self): return unicode(self).encode('utf-8') + def get_file_property(self, property): + """ + Return a list of the selected property + """ + print(self._special_comments) + if property in self._special_comments: + return self._special_comments[property] + else: + return None + + def get_only_file_property(self, property): + """ + Return a single element of the selected property + """ + elements = self.get_file_property(property) + if elements: + return elements[0] + else: + return None + + class OrgRootNode(OrgBaseNode): diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index 1f43e79..f4fe3a4 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -94,3 +94,38 @@ def test_add_custom_todo_keys(): root = loads(content, filename, env) assert root.env.all_todo_keys == ['CUSTOM_TODO', 'COMMENT_TODO', 'CUSTOM_DONE', 'COMMENT_DONE'] + +def test_get_custom_property(): + filename = '' # default for loads + content = """#+TITLE: Test title + * Node 1 + test 1 + * Node 2 + test 2 + """ + + env = OrgEnv(filename=filename) + + # after parsing, all keys are set + root = loads(content, filename, env) + assert root.get_file_property('TITLE') == ['Test title'] + assert root.get_only_file_property('TITLE') == 'Test title' + +def test_get_custom_property_multivalued(): + filename = '' # default for loads + content = """#+TITLE: Test + #+OTHER: Test title + #+TITLE: alternate title + + * Node 1 + test 1 + * Node 2 + test 2 + """ + + env = OrgEnv(filename=filename) + + # after parsing, all keys are set + root = loads(content, filename, env) + assert root.get_file_property('TITLE') == ['Test', 'alternate title'] + assert root.get_only_file_property('TITLE') == 'Test' From b64dd8f7e180be201e44a4e92f059e4d399a9e83 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 2 Nov 2020 20:36:07 +0000 Subject: [PATCH 50/90] file properties getter: cleanup (original PR: https://github.com/karlicoss/orgparse/pull/23) --- orgparse/node.py | 31 ++++++++++++++++--------------- orgparse/tests/test_misc.py | 34 +++++++++++++++++----------------- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index 599120f..d025576 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -154,8 +154,10 @@ def parse_comment(line): >>> parse_comment('# not a special comment') # None """ - if re.match(r'\s*#\+', line): - comment = re.split(r':\s+', re.split(r'\s*#\+',line)[1]) + match = re.match(r'\s*#\+', line) + if match: + end = match.end(0) + comment = line[end:].split(':') if len(comment) == 2: return (comment[0], comment[1].strip()) @@ -725,26 +727,25 @@ def __unicode__(self): def __str__(self): return unicode(self).encode('utf-8') - def get_file_property(self, property): + # todo hmm, not sure if it really belongs here and not to OrgRootNode? + def get_file_property_list(self, property): """ Return a list of the selected property """ - print(self._special_comments) - if property in self._special_comments: - return self._special_comments[property] - else: - return None + vals = self._special_comments.get(property, None) + return [] if vals is None else vals - def get_only_file_property(self, property): + def get_file_property(self, property): """ - Return a single element of the selected property + Return a single element of the selected property or None if it doesn't exist """ - elements = self.get_file_property(property) - if elements: - return elements[0] - else: + vals = self._special_comments.get(property, None) + if vals is None: return None - + elif len(vals) == 1: + return vals[0] + else: + raise RuntimeError('Multiple values for property {}: {}'.format(property, vals)) class OrgRootNode(OrgBaseNode): diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index f4fe3a4..0308883 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -95,25 +95,23 @@ def test_add_custom_todo_keys(): assert root.env.all_todo_keys == ['CUSTOM_TODO', 'COMMENT_TODO', 'CUSTOM_DONE', 'COMMENT_DONE'] -def test_get_custom_property(): - filename = '' # default for loads - content = """#+TITLE: Test title +def test_get_file_property(): + content = """#+TITLE: Test title * Node 1 test 1 * Node 2 test 2 """ - env = OrgEnv(filename=filename) - # after parsing, all keys are set - root = loads(content, filename, env) - assert root.get_file_property('TITLE') == ['Test title'] - assert root.get_only_file_property('TITLE') == 'Test title' - -def test_get_custom_property_multivalued(): - filename = '' # default for loads - content = """#+TITLE: Test + root = loads(content) + assert root.get_file_property('Nosuchproperty') is None + assert root.get_file_property_list('TITLE') == ['Test title'] + assert root.get_file_property('TITLE') == 'Test title' + assert root.get_file_property_list('Nosuchproperty') == [] + +def test_get_file_property_multivalued(): + content = """ #+TITLE: Test #+OTHER: Test title #+TITLE: alternate title @@ -123,9 +121,11 @@ def test_get_custom_property_multivalued(): test 2 """ - env = OrgEnv(filename=filename) - # after parsing, all keys are set - root = loads(content, filename, env) - assert root.get_file_property('TITLE') == ['Test', 'alternate title'] - assert root.get_only_file_property('TITLE') == 'Test' + root = loads(content) + import pytest + + assert root.get_file_property_list('TITLE') == ['Test', 'alternate title'] + with pytest.raises(RuntimeError): + # raises because there are multiple of them + root.get_file_property('TITLE') From 30671890e38db0c7becab12dfda6ff7d97496a8b Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 1 Nov 2020 20:31:38 +0000 Subject: [PATCH 51/90] add .body property for root node --- orgparse/__init__.py | 1 + orgparse/node.py | 66 +++++++++++++++++++------------------ orgparse/tests/test_misc.py | 12 +++++++ 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 7930945..cb7dea8 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -127,6 +127,7 @@ def load(path, env=None): :rtype: :class:`orgparse.node.OrgRootNode` """ + path = str(path) # in case of pathlib.Path if isinstance(path, basestring): orgfile = codecs.open(path, encoding='utf8') filename = path diff --git a/orgparse/node.py b/orgparse/node.py index d025576..d9bfcbe 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -353,6 +353,8 @@ class OrgBaseNode(Sequence): 5 """ + _body_lines: List[str] # set by the child classes + def __init__(self, env, index=None) -> None: """ Create an :class:`OrgBaseNode` object. @@ -703,6 +705,30 @@ def shallow_tags(self) -> Set[str]: """ return self._get_tags(inher=False) + @staticmethod + def _get_text(text, format='plain'): + if format == 'plain': + return to_plain_text(text) + elif format == 'raw': + return text + else: + raise ValueError('format={0} is not supported.'.format(format)) + + def get_body(self, format='plain') -> str: + """ + Return a string of body text. + + See also: :meth:`get_heading`. + + """ + return self._get_text( + '\n'.join(self._body_lines), format) if self._lines else '' + + @property + def body(self) -> str: + """Alias of ``.get_body(format='plain')``.""" + return self.get_body() + def is_root(self): """ Return ``True`` when it is a root node. @@ -757,7 +783,11 @@ class OrgRootNode(OrgBaseNode): """ - # getter + @property + def _body_lines(self) -> List[str]: # type: ignore[override] + # todo hacky.. + # for root node, the body is whatever is before the first node + return self._lines @property def level(self): @@ -766,8 +796,6 @@ def level(self): def get_parent(self, max_level=None): return None - # misc - def is_root(self): return True @@ -909,17 +937,6 @@ def _iparse_repeated_tasks(self, ilines: Iterator[str]) -> Iterator[str]: \[ (?P [^\]]+) \]''', re.VERBOSE) - # getter - - @staticmethod - def _get_text(text, format='plain'): - if format == 'plain': - return to_plain_text(text) - elif format == 'raw': - return text - else: - raise ValueError('format={0} is not supported.'.format(format)) - def get_heading(self, format='plain'): """ Return a string of head text without tags and TODO keywords. @@ -942,26 +959,11 @@ def get_heading(self, format='plain'): """ return self._get_text(self._heading, format) - def get_body(self, format='plain'): - """ - Return a string of body text. - - See also: :meth:`get_heading`. - - """ - return self._get_text( - '\n'.join(self._body_lines), format) if self._lines else '' - @property - def heading(self): + def heading(self) -> str: """Alias of ``.get_heading(format='plain')``.""" return self.get_heading() - @property - def body(self): - """Alias of ``.get_body(format='plain')``.""" - return self.get_body() - @property def level(self): return self._level @@ -1022,7 +1024,7 @@ def todo(self) -> Optional[str]: """ return self._todo - def get_property(self, key, val=None): + def get_property(self, key, val=None) -> Optional[PropertyValue]: """ Return property named ``key`` if exists or ``val`` otherwise. @@ -1036,7 +1038,7 @@ def get_property(self, key, val=None): return self._properties.get(key, val) @property - def properties(self): + def properties(self) -> Dict[str, PropertyValue]: """ Node properties as a dictionary. diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index 0308883..61868f7 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -14,6 +14,18 @@ def test_empty_heading() -> None: assert h.tags == {'sometag'} +def test_root() -> None: + root = loads(''' +#+STARTUP: hidestars +Whatever +# comment +* heading 1 + '''.strip()) + assert len(root.children) == 1 + # todo not sure if should strip special comments?? + assert root.body.endswith('Whatever\n# comment') + + def test_stars(): # https://github.com/karlicoss/orgparse/issues/7#issuecomment-533732660 root = loads(""" From 5e0921c12a1ca9f856c9e651aa0df5bb51fbccd2 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 6 Nov 2020 01:12:36 +0000 Subject: [PATCH 52/90] initial support for parsing tables https://github.com/karlicoss/orgparse/issues/8 --- .github/workflows/main.yml | 4 +- orgparse/extra.py | 74 +++++++++++++++++++++++++++++++++++++ orgparse/tests/test_rich.py | 63 +++++++++++++++++++++++++++++++ tox.ini | 4 +- 4 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 orgparse/extra.py create mode 100644 orgparse/tests/test_rich.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eef4b29..f95e198 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,8 +42,8 @@ jobs: - uses: actions/upload-artifact@v2 with: - name: .mypy-coverage_${{ matrix.platform }}_${{ matrix.python-version }} - path: .mypy-coverage/ + name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} + path: .coverage.mypy/ pypi: diff --git a/orgparse/extra.py b/orgparse/extra.py new file mode 100644 index 0000000..669d566 --- /dev/null +++ b/orgparse/extra.py @@ -0,0 +1,74 @@ +import re +from typing import List, Sequence, Dict, Iterator, Iterable, Union, Optional + + +RE_TABLE_SEPARATOR = re.compile(r'\s*\|(\-+\+)*\-+\|') +STRIP_CELL_WHITESPACE = True + + +Row = Sequence[str] + +class Table: + def __init__(self, lines: List[str]) -> None: + self._lines = lines + + @property + def blocks(self) -> Iterator[Sequence[Row]]: + group: List[Row] = [] + first = True + for r in self._pre_rows(): + if r is None: + if not first or len(group) > 0: + yield group + first = False + group = [] + else: + group.append(r) + if len(group) > 0: + yield group + + def __iter__(self) -> Iterator[Row]: + return self.rows + + @property + def rows(self) -> Iterator[Row]: + for r in self._pre_rows(): + if r is not None: + yield r + + def _pre_rows(self) -> Iterator[Optional[Row]]: + for l in self._lines: + if RE_TABLE_SEPARATOR.match(l): + yield None + else: + pr = l.strip().strip('|').split('|') + if STRIP_CELL_WHITESPACE: + pr = [x.strip() for x in pr] + yield pr + # TODO use iparse helper? + + @property + def as_dicts(self) -> 'AsDictHelper': + bl = list(self.blocks) + if len(bl) != 2: + raise RuntimeError('Need two-block table to non-ambiguously guess column names') + hrows = bl[0] + if len(hrows) != 1: + raise RuntimeError(f'Need single row heading to guess column names, got: {hrows}') + columns = hrows[0] + assert len(set(columns)) == len(columns), f'Duplicate column names: {columns}' + return AsDictHelper( + columns=columns, + rows=bl[1], + ) + + + +class AsDictHelper: + def __init__(self, columns: Sequence[str], rows: Sequence[Row]) -> None: + self.columns = columns + self._rows = rows + + def __iter__(self) -> Iterator[Dict[str, str]]: + for x in self._rows: + yield {k: v for k, v in zip(self.columns, x)} diff --git a/orgparse/tests/test_rich.py b/orgparse/tests/test_rich.py new file mode 100644 index 0000000..787e91e --- /dev/null +++ b/orgparse/tests/test_rich.py @@ -0,0 +1,63 @@ +''' +Tests for rich formatting: tables etc. +''' +from .. import load, loads +from ..extra import Table + +import pytest # type: ignore + + +def test_table() -> None: + root = loads(''' +| | | | +| | "heading" | | +| | | | +|-------+-----------+-----| +| reiwf | fef | | +|-------+-----------+-----| +|-------+-----------+-----| +| aba | caba | 123 | +| yeah | | X | + + |------------------------+-------| + | when | count | + | datetime | int | + |------------------------+-------| + | | -1 | + | [2020-11-05 Thu 23:44] | | + | [2020-11-06 Fri 01:00] | 1 | + |------------------------+-------| + +some irrelevant text + +| simple | +|--------| +| value1 | +| value2 | + ''') + + # FIXME need to parse properly + t1 = Table(root._lines[1:10]) + t2 = Table(root._lines[11:19]) + t3 = Table(root._lines[22:26]) + + assert ilen(t1.blocks) == 4 + assert list(t1.blocks)[2] == [] + assert ilen(t1.rows) == 6 + + with pytest.raises(RuntimeError): + list(t1.as_dicts) # not sure what should it be + + assert ilen(t2.blocks) == 2 + assert ilen(t2.rows) == 5 + assert list(t2.rows)[3] == ['[2020-11-05 Thu 23:44]', ''] + + + assert ilen(t3.blocks) == 2 + assert list(t3.rows) == [['simple'], ['value1'], ['value2']] + assert t3.as_dicts.columns == ['simple'] + assert list(t3.as_dicts) == [{'simple': 'value1'}, {'simple': 'value2'}] + + +def ilen(x) -> int: + return len(list(x)) diff --git a/tox.ini b/tox.ini index 1a07c67..a3a312f 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,6 @@ commands = pip install -e .[linting] python -m mypy orgparse \ # txt report is a bit more convenient to view on CI - --txt-report .mypy-coverage \ - --html-report .mypy-coverage \ + --txt-report .coverage.mypy \ + --html-report .coverage.mypy \ {posargs} From 731606eda68324cc27f9b907568b6f25cf6f156d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 6 Nov 2020 01:59:38 +0000 Subject: [PATCH 53/90] initial support for rich format in body, extract tables --- orgparse/extra.py | 44 ++++++++++++++++++++++++++++++++++++- orgparse/node.py | 8 +++++++ orgparse/tests/test_rich.py | 28 ++++++++++++++++++++++- 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/orgparse/extra.py b/orgparse/extra.py index 669d566..f9aab7a 100644 --- a/orgparse/extra.py +++ b/orgparse/extra.py @@ -3,6 +3,7 @@ RE_TABLE_SEPARATOR = re.compile(r'\s*\|(\-+\+)*\-+\|') +RE_TABLE_ROW = re.compile(r'\s*\|([^|]+)+\|') STRIP_CELL_WHITESPACE = True @@ -63,7 +64,6 @@ def as_dicts(self) -> 'AsDictHelper': ) - class AsDictHelper: def __init__(self, columns: Sequence[str], rows: Sequence[Row]) -> None: self.columns = columns @@ -72,3 +72,45 @@ def __init__(self, columns: Sequence[str], rows: Sequence[Row]) -> None: def __iter__(self) -> Iterator[Dict[str, str]]: for x in self._rows: yield {k: v for k, v in zip(self.columns, x)} + + +class Gap: + # todo later, add indices etc + pass + + +Rich = Union[Table, Gap] +def to_rich_text(text: str) -> Iterator[Rich]: + ''' + Convert an org-mode text into a 'rich' text, e.g. tables/lists/etc, interleaved by gaps. + NOTE: you shouldn't rely on the number of items returned by this function, + it might change in the future when more types are supported. + + At the moment only tables are supported. + ''' + lines = text.splitlines(keepends=True) + group: List[str] = [] + last = Gap + def emit() -> Rich: + nonlocal group, last + if last is Gap: + res = Gap() + elif last is Table: + res = Table(group) # type: ignore + else: + raise RuntimeError(f'Unexpected type {last}') + group = [] + return res + + for line in lines: + if RE_TABLE_ROW.match(line) or RE_TABLE_SEPARATOR.match(line): + cur = Table + else: + cur = Gap # type: ignore + if cur is not last: + if len(group) > 0: + yield emit() + last = cur # type: ignore + group.append(line) + if len(group) > 0: + yield emit() diff --git a/orgparse/node.py b/orgparse/node.py index d9bfcbe..c21386f 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -8,6 +8,7 @@ from .date import OrgDate, OrgDateClock, OrgDateRepeatedTask, parse_sdc from .inline import to_plain_text +from .extra import to_rich_text, Rich from .utils.py3compat import PY3, unicode @@ -711,6 +712,8 @@ def _get_text(text, format='plain'): return to_plain_text(text) elif format == 'raw': return text + elif format == 'rich': + return to_rich_text(text) else: raise ValueError('format={0} is not supported.'.format(format)) @@ -729,6 +732,11 @@ def body(self) -> str: """Alias of ``.get_body(format='plain')``.""" return self.get_body() + @property + def body_rich(self) -> Iterator[Rich]: + r = self.get_body(format='rich') + return cast(Iterator[Rich], r) # meh.. + def is_root(self): """ Return ``True`` when it is a root node. diff --git a/orgparse/tests/test_rich.py b/orgparse/tests/test_rich.py index 787e91e..68fab40 100644 --- a/orgparse/tests/test_rich.py +++ b/orgparse/tests/test_rich.py @@ -36,7 +36,8 @@ def test_table() -> None: | value2 | ''') - # FIXME need to parse properly + [gap1, t1, gap2, t2, gap3, t3, gap4] = root.body_rich + t1 = Table(root._lines[1:10]) t2 = Table(root._lines[11:19]) t3 = Table(root._lines[22:26]) @@ -59,5 +60,30 @@ def test_table() -> None: assert list(t3.as_dicts) == [{'simple': 'value1'}, {'simple': 'value2'}] +def test_table_2() -> None: + root = loads(''' +* item + +#+tblname: something +| date | value | comment | +|----------------------+-------+-------------------------------| +| 14.04.17 | 11 | aaaa | +| May 26 2017 08:00 | 12 | what + about + pluses? | +| May 26 09:00 - 10:00 | 13 | time is | + + some comment + +#+BEGIN_SRC python :var fname="plot.png" :var table=something :results file +fig.savefig(fname) +return fname +#+END_SRC + +#+RESULTS: +[[file:plot.png]] +''') + [_, t, _] = root.children[0].body_rich + assert ilen(t.as_dicts) == 3 + + def ilen(x) -> int: return len(list(x)) From ab521ce9850b0c410c05c35880424db47c84720d Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 6 Nov 2020 02:52:57 +0000 Subject: [PATCH 54/90] add heading to OrgRootNode (always empty) --- orgparse/node.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/orgparse/node.py b/orgparse/node.py index c21386f..b94735b 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -737,6 +737,10 @@ def body_rich(self) -> Iterator[Rich]: r = self.get_body(format='rich') return cast(Iterator[Rich], r) # meh.. + @property + def heading(self) -> str: + raise NotImplementedError + def is_root(self): """ Return ``True`` when it is a root node. @@ -797,6 +801,10 @@ def _body_lines(self) -> List[str]: # type: ignore[override] # for root node, the body is whatever is before the first node return self._lines + @property + def heading(self) -> str: + return '' + @property def level(self): return 0 From 65465ad8769af191a21a29c7c09590343a583ad3 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 6 Nov 2020 06:00:47 +0000 Subject: [PATCH 55/90] treat filetags as tags + make file properties case insensitive --- orgparse/node.py | 26 +++++++++++++++++--------- orgparse/tests/test_misc.py | 19 +++++++++++++++++-- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index b94735b..489f21b 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1,6 +1,6 @@ import re import itertools -from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict, Set +from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict, Set, Sequence try: from collections.abc import Sequence except ImportError: @@ -146,21 +146,24 @@ def parse_property(line: str) -> Tuple[Optional[str], Optional[PropertyValue]]: RE_PROP = re.compile(r'^\s*:(.*?):\s*(.*?)\s*$') -def parse_comment(line): +def parse_comment(line: str): # -> Optional[Tuple[str, Sequence[str]]]: # todo wtf?? it says 'ABCMeta isn't subscriptable??' """ Parse special comment such as ``#+SEQ_TODO`` >>> parse_comment('#+SEQ_TODO: TODO | DONE') - ('SEQ_TODO', 'TODO | DONE') + ('SEQ_TODO', ['TODO | DONE']) >>> parse_comment('# not a special comment') # None + >>> parse_comment('#+FILETAGS: :tag1:tag2:') + ('FILETAGS', ['tag1', 'tag2']) """ match = re.match(r'\s*#\+', line) if match: end = match.end(0) comment = line[end:].split(':') - if len(comment) == 2: - return (comment[0], comment[1].strip()) + if len(comment) >= 2: + return (comment[0], [c.strip() for c in comment[1:] if len(c.strip()) > 0]) + return None def parse_seq_todo(line): @@ -642,8 +645,9 @@ def _parse_comments(self): for line in self._lines: parsed = parse_comment(line) if parsed: - (key, val) = parsed - special_comments.setdefault(key, []).append(val) + (key, vals) = parsed + key = key.upper() # case insensitive, so keep as uppercase + special_comments.setdefault(key, []).extend(vals) self._special_comments = special_comments # parse TODO keys and store in OrgEnv for todokey in ['TODO', 'SEQ_TODO', 'TYP_TODO']: @@ -770,14 +774,14 @@ def get_file_property_list(self, property): """ Return a list of the selected property """ - vals = self._special_comments.get(property, None) + vals = self._special_comments.get(property.upper(), None) return [] if vals is None else vals def get_file_property(self, property): """ Return a single element of the selected property or None if it doesn't exist """ - vals = self._special_comments.get(property, None) + vals = self._special_comments.get(property.upper(), None) if vals is None: return None elif len(vals) == 1: @@ -805,6 +809,10 @@ def _body_lines(self) -> List[str]: # type: ignore[override] def heading(self) -> str: return '' + def _get_tags(self, inher=False) -> Set[str]: + filetags = self.get_file_property_list('FILETAGS') + return set(filetags) + @property def level(self): return 0 diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index 61868f7..37128a5 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -24,6 +24,7 @@ def test_root() -> None: assert len(root.children) == 1 # todo not sure if should strip special comments?? assert root.body.endswith('Whatever\n# comment') + assert root.heading == '' def test_stars(): @@ -119,13 +120,14 @@ def test_get_file_property(): root = loads(content) assert root.get_file_property('Nosuchproperty') is None assert root.get_file_property_list('TITLE') == ['Test title'] - assert root.get_file_property('TITLE') == 'Test title' + # also it's case insensitive + assert root.get_file_property('title') == 'Test title' assert root.get_file_property_list('Nosuchproperty') == [] def test_get_file_property_multivalued(): content = """ #+TITLE: Test #+OTHER: Test title - #+TITLE: alternate title + #+title: alternate title * Node 1 test 1 @@ -141,3 +143,16 @@ def test_get_file_property_multivalued(): with pytest.raises(RuntimeError): # raises because there are multiple of them root.get_file_property('TITLE') + +def test_filetags_are_tags() -> None: + content = ''' +#+FILETAGS: :f1:f2: + +* heading :h1: +** child :f2: + '''.strip() + root = loads(content) + # breakpoint() + assert root.tags == {'f1', 'f2'} + child = root.children[0].children[0] + assert child.tags == {'f1', 'f2', 'h1'} From 51fe724d35a7105d83f4e72bb728f4fbbb2fe2ab Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 6 Nov 2020 20:09:44 +0000 Subject: [PATCH 56/90] fix: py.typed typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f3a6a51..5f02a7d 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def main(): packages=[pkg, *subpkgs], package_data={ - 'pkg': ['py.typed'], # todo need the rest as well?? + pkg: ['py.typed'], # todo need the rest as well?? 'orgparse.tests.data': ['*.org'], }, From ce8f6ccf60d6dca60c5f03890d730676a5d1ac21 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 6 Dec 2020 23:09:39 +0000 Subject: [PATCH 57/90] only split filetags by ':' --- orgparse/node.py | 11 +++++++++-- orgparse/tests/test_misc.py | 12 +++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index 489f21b..b8628e1 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -160,9 +160,16 @@ def parse_comment(line: str): # -> Optional[Tuple[str, Sequence[str]]]: # todo match = re.match(r'\s*#\+', line) if match: end = match.end(0) - comment = line[end:].split(':') + comment = line[end:].split(':', maxsplit=1) if len(comment) >= 2: - return (comment[0], [c.strip() for c in comment[1:] if len(c.strip()) > 0]) + key = comment[0] + value = comment[1].strip() + if key.upper() == 'FILETAGS': + # just legacy behaviour; it seems like filetags is the only one that separated by ':' + # see https://orgmode.org/org.html#In_002dbuffer-Settings + return (key, [c.strip() for c in value.split(':') if len(c.strip()) > 0]) + else: + return (key, [value]) return None diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index 37128a5..6f01757 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -108,8 +108,8 @@ def test_add_custom_todo_keys(): assert root.env.all_todo_keys == ['CUSTOM_TODO', 'COMMENT_TODO', 'CUSTOM_DONE', 'COMMENT_DONE'] -def test_get_file_property(): - content = """#+TITLE: Test title +def test_get_file_property() -> None: + content = """#+TITLE: Test: title * Node 1 test 1 * Node 2 @@ -119,12 +119,13 @@ def test_get_file_property(): # after parsing, all keys are set root = loads(content) assert root.get_file_property('Nosuchproperty') is None - assert root.get_file_property_list('TITLE') == ['Test title'] + assert root.get_file_property_list('TITLE') == ['Test: title'] # also it's case insensitive - assert root.get_file_property('title') == 'Test title' + assert root.get_file_property('title') == 'Test: title' assert root.get_file_property_list('Nosuchproperty') == [] -def test_get_file_property_multivalued(): + +def test_get_file_property_multivalued() -> None: content = """ #+TITLE: Test #+OTHER: Test title #+title: alternate title @@ -144,6 +145,7 @@ def test_get_file_property_multivalued(): # raises because there are multiple of them root.get_file_property('TITLE') + def test_filetags_are_tags() -> None: content = ''' #+FILETAGS: :f1:f2: From 45b366eb5c23e4149d8bc7a319f10dc74888fe51 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Fri, 8 Jan 2021 07:07:18 +0000 Subject: [PATCH 58/90] fix regression in orgparse.load method for file-like objects broken in https://github.com/karlicoss/orgparse/commit/30671890e38db0c7becab12dfda6ff7d97496a8b see https://github.com/karlicoss/orgparse/issues/32 --- orgparse/__init__.py | 20 +++++++++++--------- orgparse/tests/test_misc.py | 11 +++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index cb7dea8..b3a42d0 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -107,9 +107,11 @@ # [[[end]]] import codecs -from typing import Iterable +from pathlib import Path +from typing import Iterable, Union, Optional, TextIO -from .node import parse_lines, OrgNode # todo basenode?? + +from .node import parse_lines, OrgEnv, OrgNode # todo basenode?? from .utils.py3compat import basestring __author__ = 'Takafumi Arakaki, Dmitrii Gerasimov' @@ -117,7 +119,7 @@ __all__ = ["load", "loads", "loadi"] -def load(path, env=None): +def load(path: Union[str, Path, TextIO], env: Optional[OrgEnv]=None) -> OrgNode: """ Load org-mode document from a file. @@ -127,10 +129,10 @@ def load(path, env=None): :rtype: :class:`orgparse.node.OrgRootNode` """ - path = str(path) # in case of pathlib.Path - if isinstance(path, basestring): - orgfile = codecs.open(path, encoding='utf8') - filename = path + orgfile: TextIO + if isinstance(path, (str, Path)): + orgfile = codecs.open(str(path), encoding='utf8') + filename = str(path) else: orgfile = path filename = path.name if hasattr(path, 'name') else '' @@ -138,7 +140,7 @@ def load(path, env=None): filename=filename, env=env) -def loads(string: str, filename='', env=None) -> OrgNode: +def loads(string: str, filename: str='', env: Optional[OrgEnv]=None) -> OrgNode: """ Load org-mode document from a string. @@ -148,7 +150,7 @@ def loads(string: str, filename='', env=None) -> OrgNode: return loadi(string.splitlines(), filename=filename, env=env) -def loadi(lines: Iterable[str], filename='', env=None) -> OrgNode: +def loadi(lines: Iterable[str], filename: str='', env: Optional[OrgEnv]=None) -> OrgNode: """ Load org-mode document from an iterative object. diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index 6f01757..732944d 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -158,3 +158,14 @@ def test_filetags_are_tags() -> None: assert root.tags == {'f1', 'f2'} child = root.children[0].children[0] assert child.tags == {'f1', 'f2', 'h1'} + + +def test_load_filelike() -> None: + import io + stream = io.StringIO(''' +* heading1 +* heading 2 +''') + root = load(stream) + assert len(root.children) == 2 + assert root.env.filename == '' From 2590d3d40da62e92b1a247e6c8b1c3bd89668ede Mon Sep 17 00:00:00 2001 From: Johnson Date: Wed, 21 Apr 2021 02:24:50 +0800 Subject: [PATCH 59/90] Allow parsing for all duration formats --- orgparse/__init__.py | 2 +- orgparse/node.py | 96 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index b3a42d0..899954b 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -96,7 +96,7 @@ >>> node.tags == set(['TAG']) True >>> node.get_property('Effort') -60 +60.0 >>> node.get_property('UndefinedProperty') # returns None >>> node.get_property('OtherProperty') 'some text' diff --git a/orgparse/node.py b/orgparse/node.py index b8628e1..c033e87 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -120,7 +120,7 @@ def parse_heading_priority(heading): RE_HEADING_PRIORITY = re.compile(r'^\s*\[#([A-Z0-9])\] ?(.*)$') -PropertyValue = Union[str, int] +PropertyValue = Union[str, float] def parse_property(line: str) -> Tuple[Optional[str], Optional[PropertyValue]]: """ Get property from given string. @@ -128,23 +128,107 @@ def parse_property(line: str) -> Tuple[Optional[str], Optional[PropertyValue]]: >>> parse_property(':Some_property: some value') ('Some_property', 'some value') >>> parse_property(':Effort: 1:10') - ('Effort', 70) + ('Effort', 70.0) """ prop_key = None - prop_val: Optional[Union[str, int]] = None + prop_val: Optional[Union[str, float]] = None match = RE_PROP.search(line) if match: prop_key = match.group(1) prop_val = match.group(2) if prop_key == 'Effort': - (h, m) = prop_val.split(":", 2) - if h.isdigit() and m.isdigit(): - prop_val = int(h) * 60 + int(m) + prop_val = parse_duration_to_minutes(prop_val) return (prop_key, prop_val) RE_PROP = re.compile(r'^\s*:(.*?):\s*(.*?)\s*$') +def parse_duration_to_minutes(duration: str) -> float: + """ + Parse duration minutes from given string. + The following code fully mimics the 'org-duration-to-minutes' function in org mode: + https://github.com/emacs-mirror/emacs/blob/master/lisp/org/org-duration.el + + >>> parse_property(':Effort: 3:12') + ('Effort', 192.0) + >>> parse_property(':Effort: 1:23:45') + ('Effort', 83.75) + >>> parse_property(':Effort: 1y 3d 3h 4min') + ('Effort', 530464.0) + >>> parse_property(':Effort: 1d3h5min') + ('Effort', 1625.0) + >>> parse_property(':Effort: 3d 13:35') + ('Effort', 5135.0) + >>> parse_property(':Effort: 2.35h') + ('Effort', 141.0) + >>> parse_property(':Effort: 10') + ('Effort', 10.0) + >>> parse_property(':Effort: 10.') + ('Effort', 10.0) + >>> parse_property(':Effort: 1 h') + ('Effort', 60.0) + >>> parse_property(':Effort: ') + ('Effort', 0.0) + """ + + # Conversion factor to minutes for a duration. + ORG_DURATION_UNITS = { + "min": 1, + "h": 60, + "d": 60 * 24, + "w": 60 * 24 * 7, + "m": 60 * 24 * 30, + "y": 60 * 24 * 365.25, + } + # Regexp matching for all units. + ORG_DURATION_UNITS_RE = r'(%s)' % r'|'.join(ORG_DURATION_UNITS.keys()) + # Regexp matching a duration expressed with H:MM or H:MM:SS format. + # Hours can use any number of digits. + ORG_DURATION_H_MM_RE = r'[ \t]*[0-9]+(?::[0-9]{2}){1,2}[ \t]*' + RE_ORG_DURATION_H_MM = re.compile(ORG_DURATION_H_MM_RE) + # Regexp matching a duration with an unit. + # Allowed units are defined in ORG_DURATION_UNITS. + # Match group 1 contains the bare number. + # Match group 2 contains the unit. + ORG_DURATION_UNIT_RE = r'([0-9]+(?:[.][0-9]*)?)[ \t]*' + ORG_DURATION_UNITS_RE + RE_ORG_DURATION_UNIT = re.compile(ORG_DURATION_UNIT_RE) + # Regexp matching a duration expressed with units. + # Allowed units are defined in ORG_DURATION_UNITS. + ORG_DURATION_FULL_RE = r'(?:[ \t]*%s)+[ \t]*' % ORG_DURATION_UNIT_RE + RE_ORG_DURATION_FULL = re.compile(ORG_DURATION_FULL_RE) + # Regexp matching a duration expressed with units and H:MM or H:MM:SS format. + # Allowed units are defined in ORG_DURATION_UNITS. + # Match group A contains units part. + # Match group B contains H:MM or H:MM:SS part. + ORG_DURATION_MIXED_RE = r'(?P([ \t]*%s)+)[ \t]*(?P[0-9]+(?::[0-9][0-9]){1,2})[ \t]*' % ORG_DURATION_UNIT_RE + RE_ORG_DURATION_MIXED = re.compile(ORG_DURATION_MIXED_RE) + # Regexp matching float numbers. + RE_FLOAT = re.compile(r'[0-9]+([.][0-9]*)?') + + match: Optional[re.Match[str]] + if duration == "": + return 0.0 + if isinstance(duration, float): + return float(duration) + if RE_ORG_DURATION_H_MM.fullmatch(duration): + hours, minutes, *seconds_ = map(float, duration.split(":")) + seconds = seconds_[0] if seconds_ else 0 + return seconds / 60.0 + minutes + 60 * hours + if RE_ORG_DURATION_FULL.fullmatch(duration): + minutes = 0 + for match in RE_ORG_DURATION_UNIT.finditer(duration): + value = float(match.group(1)) + unit = match.group(2) + minutes += value * ORG_DURATION_UNITS[unit] + return float(minutes) + match = RE_ORG_DURATION_MIXED.fullmatch(duration) + if match: + units_part = match.groupdict()['A'] + hms_part = match.groupdict()['B'] + return parse_duration_to_minutes(units_part) + parse_duration_to_minutes(hms_part) + if RE_FLOAT.fullmatch(duration): + return float(duration) + raise ValueError("Invalid duration format %s" % duration) def parse_comment(line: str): # -> Optional[Tuple[str, Sequence[str]]]: # todo wtf?? it says 'ABCMeta isn't subscriptable??' """ From f05e9032c10ae6a8f9a7e753cd5492c6ff763491 Mon Sep 17 00:00:00 2001 From: Johnson Date: Wed, 21 Apr 2021 20:50:51 +0800 Subject: [PATCH 60/90] Support int duration for backward compatibility --- orgparse/__init__.py | 2 +- orgparse/node.py | 82 ++++++++++++++++++++++++++++++-------------- 2 files changed, 57 insertions(+), 27 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 899954b..b3a42d0 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -96,7 +96,7 @@ >>> node.tags == set(['TAG']) True >>> node.get_property('Effort') -60.0 +60 >>> node.get_property('UndefinedProperty') # returns None >>> node.get_property('OtherProperty') 'some text' diff --git a/orgparse/node.py b/orgparse/node.py index c033e87..b95cf20 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -120,7 +120,7 @@ def parse_heading_priority(heading): RE_HEADING_PRIORITY = re.compile(r'^\s*\[#([A-Z0-9])\] ?(.*)$') -PropertyValue = Union[str, float] +PropertyValue = Union[str, int, float] def parse_property(line: str) -> Tuple[Optional[str], Optional[PropertyValue]]: """ Get property from given string. @@ -128,11 +128,11 @@ def parse_property(line: str) -> Tuple[Optional[str], Optional[PropertyValue]]: >>> parse_property(':Some_property: some value') ('Some_property', 'some value') >>> parse_property(':Effort: 1:10') - ('Effort', 70.0) + ('Effort', 70) """ prop_key = None - prop_val: Optional[Union[str, float]] = None + prop_val: Optional[Union[str, int, float]] = None match = RE_PROP.search(line) if match: prop_key = match.group(1) @@ -143,32 +143,62 @@ def parse_property(line: str) -> Tuple[Optional[str], Optional[PropertyValue]]: RE_PROP = re.compile(r'^\s*:(.*?):\s*(.*?)\s*$') -def parse_duration_to_minutes(duration: str) -> float: +def parse_duration_to_minutes(duration: str) -> Union[float, int]: """ Parse duration minutes from given string. - The following code fully mimics the 'org-duration-to-minutes' function in org mode: + Convert to integer if number has no decimal points + + >>> parse_duration_to_minutes('3:12') + 192 + >>> parse_duration_to_minutes('1:23:45') + 83.75 + >>> parse_duration_to_minutes('1y 3d 3h 4min') + 530464 + >>> parse_duration_to_minutes('1d3h5min') + 1625 + >>> parse_duration_to_minutes('3d 13:35') + 5135 + >>> parse_duration_to_minutes('2.35h') + 141 + >>> parse_duration_to_minutes('10') + 10 + >>> parse_duration_to_minutes('10.') + 10 + >>> parse_duration_to_minutes('1 h') + 60 + >>> parse_duration_to_minutes('') + 0 + """ + + minutes = parse_duration_to_minutes_float(duration) + return int(minutes) if minutes.is_integer() else minutes + +def parse_duration_to_minutes_float(duration: str) -> float: + """ + Parse duration minutes from given string. + The following code is fully compatible with the 'org-duration-to-minutes' function in org mode: https://github.com/emacs-mirror/emacs/blob/master/lisp/org/org-duration.el - >>> parse_property(':Effort: 3:12') - ('Effort', 192.0) - >>> parse_property(':Effort: 1:23:45') - ('Effort', 83.75) - >>> parse_property(':Effort: 1y 3d 3h 4min') - ('Effort', 530464.0) - >>> parse_property(':Effort: 1d3h5min') - ('Effort', 1625.0) - >>> parse_property(':Effort: 3d 13:35') - ('Effort', 5135.0) - >>> parse_property(':Effort: 2.35h') - ('Effort', 141.0) - >>> parse_property(':Effort: 10') - ('Effort', 10.0) - >>> parse_property(':Effort: 10.') - ('Effort', 10.0) - >>> parse_property(':Effort: 1 h') - ('Effort', 60.0) - >>> parse_property(':Effort: ') - ('Effort', 0.0) + >>> parse_duration_to_minutes_float('3:12') + 192.0 + >>> parse_duration_to_minutes_float('1:23:45') + 83.75 + >>> parse_duration_to_minutes_float('1y 3d 3h 4min') + 530464.0 + >>> parse_duration_to_minutes_float('1d3h5min') + 1625.0 + >>> parse_duration_to_minutes_float('3d 13:35') + 5135.0 + >>> parse_duration_to_minutes_float('2.35h') + 141.0 + >>> parse_duration_to_minutes_float('10') + 10.0 + >>> parse_duration_to_minutes_float('10.') + 10.0 + >>> parse_duration_to_minutes_float('1 h') + 60.0 + >>> parse_duration_to_minutes_float('') + 0.0 """ # Conversion factor to minutes for a duration. @@ -225,7 +255,7 @@ def parse_duration_to_minutes(duration: str) -> float: if match: units_part = match.groupdict()['A'] hms_part = match.groupdict()['B'] - return parse_duration_to_minutes(units_part) + parse_duration_to_minutes(hms_part) + return parse_duration_to_minutes_float(units_part) + parse_duration_to_minutes_float(hms_part) if RE_FLOAT.fullmatch(duration): return float(duration) raise ValueError("Invalid duration format %s" % duration) From e3eeca3229dc1bf1b3e60c86eae12e0c098706a9 Mon Sep 17 00:00:00 2001 From: Johnson Date: Wed, 21 Apr 2021 20:58:24 +0800 Subject: [PATCH 61/90] Move regex vars to global --- orgparse/node.py | 68 ++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index b95cf20..44a609e 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -201,40 +201,6 @@ def parse_duration_to_minutes_float(duration: str) -> float: 0.0 """ - # Conversion factor to minutes for a duration. - ORG_DURATION_UNITS = { - "min": 1, - "h": 60, - "d": 60 * 24, - "w": 60 * 24 * 7, - "m": 60 * 24 * 30, - "y": 60 * 24 * 365.25, - } - # Regexp matching for all units. - ORG_DURATION_UNITS_RE = r'(%s)' % r'|'.join(ORG_DURATION_UNITS.keys()) - # Regexp matching a duration expressed with H:MM or H:MM:SS format. - # Hours can use any number of digits. - ORG_DURATION_H_MM_RE = r'[ \t]*[0-9]+(?::[0-9]{2}){1,2}[ \t]*' - RE_ORG_DURATION_H_MM = re.compile(ORG_DURATION_H_MM_RE) - # Regexp matching a duration with an unit. - # Allowed units are defined in ORG_DURATION_UNITS. - # Match group 1 contains the bare number. - # Match group 2 contains the unit. - ORG_DURATION_UNIT_RE = r'([0-9]+(?:[.][0-9]*)?)[ \t]*' + ORG_DURATION_UNITS_RE - RE_ORG_DURATION_UNIT = re.compile(ORG_DURATION_UNIT_RE) - # Regexp matching a duration expressed with units. - # Allowed units are defined in ORG_DURATION_UNITS. - ORG_DURATION_FULL_RE = r'(?:[ \t]*%s)+[ \t]*' % ORG_DURATION_UNIT_RE - RE_ORG_DURATION_FULL = re.compile(ORG_DURATION_FULL_RE) - # Regexp matching a duration expressed with units and H:MM or H:MM:SS format. - # Allowed units are defined in ORG_DURATION_UNITS. - # Match group A contains units part. - # Match group B contains H:MM or H:MM:SS part. - ORG_DURATION_MIXED_RE = r'(?P([ \t]*%s)+)[ \t]*(?P[0-9]+(?::[0-9][0-9]){1,2})[ \t]*' % ORG_DURATION_UNIT_RE - RE_ORG_DURATION_MIXED = re.compile(ORG_DURATION_MIXED_RE) - # Regexp matching float numbers. - RE_FLOAT = re.compile(r'[0-9]+([.][0-9]*)?') - match: Optional[re.Match[str]] if duration == "": return 0.0 @@ -260,6 +226,40 @@ def parse_duration_to_minutes_float(duration: str) -> float: return float(duration) raise ValueError("Invalid duration format %s" % duration) +# Conversion factor to minutes for a duration. +ORG_DURATION_UNITS = { + "min": 1, + "h": 60, + "d": 60 * 24, + "w": 60 * 24 * 7, + "m": 60 * 24 * 30, + "y": 60 * 24 * 365.25, +} +# Regexp matching for all units. +ORG_DURATION_UNITS_RE = r'(%s)' % r'|'.join(ORG_DURATION_UNITS.keys()) +# Regexp matching a duration expressed with H:MM or H:MM:SS format. +# Hours can use any number of digits. +ORG_DURATION_H_MM_RE = r'[ \t]*[0-9]+(?::[0-9]{2}){1,2}[ \t]*' +RE_ORG_DURATION_H_MM = re.compile(ORG_DURATION_H_MM_RE) +# Regexp matching a duration with an unit. +# Allowed units are defined in ORG_DURATION_UNITS. +# Match group 1 contains the bare number. +# Match group 2 contains the unit. +ORG_DURATION_UNIT_RE = r'([0-9]+(?:[.][0-9]*)?)[ \t]*' + ORG_DURATION_UNITS_RE +RE_ORG_DURATION_UNIT = re.compile(ORG_DURATION_UNIT_RE) +# Regexp matching a duration expressed with units. +# Allowed units are defined in ORG_DURATION_UNITS. +ORG_DURATION_FULL_RE = r'(?:[ \t]*%s)+[ \t]*' % ORG_DURATION_UNIT_RE +RE_ORG_DURATION_FULL = re.compile(ORG_DURATION_FULL_RE) +# Regexp matching a duration expressed with units and H:MM or H:MM:SS format. +# Allowed units are defined in ORG_DURATION_UNITS. +# Match group A contains units part. +# Match group B contains H:MM or H:MM:SS part. +ORG_DURATION_MIXED_RE = r'(?P([ \t]*%s)+)[ \t]*(?P[0-9]+(?::[0-9][0-9]){1,2})[ \t]*' % ORG_DURATION_UNIT_RE +RE_ORG_DURATION_MIXED = re.compile(ORG_DURATION_MIXED_RE) +# Regexp matching float numbers. +RE_FLOAT = re.compile(r'[0-9]+([.][0-9]*)?') + def parse_comment(line: str): # -> Optional[Tuple[str, Sequence[str]]]: # todo wtf?? it says 'ABCMeta isn't subscriptable??' """ Parse special comment such as ``#+SEQ_TODO`` From 32bb63e9305bd06511cd74d514c34d5b284f1e14 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Wed, 21 Apr 2021 19:38:01 +0100 Subject: [PATCH 62/90] update ci config --- .github/workflows/main.yml | 14 ++++++++------ orgparse/node.py | 4 ++-- pytest.ini | 9 +++++++++ tox.ini | 9 +++++---- 4 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 pytest.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f95e198..cc58280 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,8 +7,10 @@ on: tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug: # P.S. fuck made up yaml DSLs. - # TODO cron? + pull_request: # needed to trigger on others' PRs + # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them". workflow_dispatch: # needed to trigger workflows manually + # todo cron? env: # useful for scripts & sometimes tests to know @@ -18,8 +20,8 @@ jobs: build: strategy: matrix: - platform: [ubuntu-latest] # macos-latest] # TODO windows-latest?? - python-version: [3.6, 3.7, 3.8] + platform: [ubuntu-latest, macos-latest] # TODO windows-latest?? + python-version: [3.6, 3.7, 3.8, 3.9] runs-on: ${{ matrix.platform }} @@ -36,7 +38,7 @@ jobs: submodules: recursive # uncomment for SSH debugging - # - uses: mxschmitt/action-tmate@v2 + # - uses: mxschmitt/action-tmate@v3 - run: .ci/run @@ -64,7 +66,7 @@ jobs: - name: 'release to test pypi' # always deploy merged master to test pypi - if: github.event.ref == 'refs/heads/master' + if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master' env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }} run: pip3 install --user wheel twine && .ci/release --test @@ -72,7 +74,7 @@ jobs: - name: 'release to pypi' # always deploy tags to release pypi # NOTE: release tags are guarded by on: push: tags on the top - if: startsWith(github.event.ref, 'refs/tags') + if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags') env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} run: pip3 install --user wheel twine && .ci/release diff --git a/orgparse/node.py b/orgparse/node.py index 44a609e..df079cd 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1,6 +1,6 @@ import re import itertools -from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict, Set, Sequence +from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict, Set, Sequence, Any try: from collections.abc import Sequence except ImportError: @@ -201,7 +201,7 @@ def parse_duration_to_minutes_float(duration: str) -> float: 0.0 """ - match: Optional[re.Match[str]] + match: Optional[Any] if duration == "": return 0.0 if isinstance(duration, float): diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..3b94dc4 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +# discover files that don't follow test_ naming. Useful to keep tests along with the source code +python_files = *.py +addopts = + -rap + --verbose + + # otherwise it won't discover doctests + --doctest-modules diff --git a/tox.ini b/tox.ini index a3a312f..b0324f6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,17 @@ [tox] minversion = 3.5 # relies on the correct version of Python installed -envlist = py3,mypy +envlist = tests,mypy [testenv] +passenv = CI CI_* + +[testenv:tests] commands = pip install -e .[testing] - python -m pytest -rap --doctest-modules --ignore-glob='**/_py3compat.py' orgparse {posargs} - + python -m pytest --ignore-glob='**/_py3compat.py' orgparse {posargs} [testenv:mypy] -skip_install = true commands = pip install -e .[linting] python -m mypy orgparse \ From a76c7d61f92d2546a4ca1e0781fe1982fd51541d Mon Sep 17 00:00:00 2001 From: "Thomas Bruckmaier (Xubuntu Laptop)" Date: Fri, 3 Sep 2021 15:34:30 +0200 Subject: [PATCH 63/90] Added support for level-0 property drawers Since org 9.4, property drawers are allowed & belong to a "level 0 headline". In orgparse, they should end up in OrgRootNode - moved property methods from OrgNode to OrgBaseNode & added parsing in OrgRootNode - removed obsolete todo, OrgRootNode correctly contains all lines before the first headline --- orgparse/node.py | 121 +++++++++++++++++++----------------- orgparse/tests/test_misc.py | 18 ++++++ 2 files changed, 81 insertions(+), 58 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index df079cd..41ceaaf 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -495,6 +495,8 @@ def __init__(self, env, index=None) -> None: # content self._lines: List[str] = [] + self._properties: Dict[str, PropertyValue] = {} + # FIXME: use `index` argument to set index. (Currently it is # done externally in `parse_lines`.) if index is not None: @@ -752,6 +754,37 @@ def root(self): return root root = parent + @property + def properties(self) -> Dict[str, PropertyValue]: + """ + Node properties as a dictionary. + + >>> from orgparse import loads + >>> root = loads(''' + ... * Node + ... :PROPERTIES: + ... :SomeProperty: value + ... :END: + ... ''') + >>> root.children[0].properties['SomeProperty'] + 'value' + + """ + return self._properties + + def get_property(self, key, val=None) -> Optional[PropertyValue]: + """ + Return property named ``key`` if exists or ``val`` otherwise. + + :arg str key: + Key of property. + + :arg val: + Default value to return. + + """ + return self._properties.get(key, val) + # parser @classmethod @@ -775,6 +808,24 @@ def _parse_comments(self): for val in special_comments.get(todokey, []): self.env.add_todo_keys(*parse_seq_todo(val)) + def _iparse_properties(self, ilines: Iterator[str]) -> Iterator[str]: + self._properties = {} + in_property_field = False + for line in ilines: + if in_property_field: + if line.find(":END:") >= 0: + break + else: + (key, val) = parse_property(line) + if key is not None and val is not None: + self._properties.update({key: val}) + elif line.find(":PROPERTIES:") >= 0: + in_property_field = True + else: + yield line + for line in ilines: + yield line + # misc @property @@ -914,18 +965,12 @@ def get_file_property(self, property): class OrgRootNode(OrgBaseNode): """ - Node to represent a file + Node to represent a file. Its body contains all lines before the first + headline See :class:`OrgBaseNode` for other available functions. - """ - @property - def _body_lines(self) -> List[str]: # type: ignore[override] - # todo hacky.. - # for root node, the body is whatever is before the first node - return self._lines - @property def heading(self) -> str: return '' @@ -944,6 +989,14 @@ def get_parent(self, max_level=None): def is_root(self): return True + # parsers + + def _parse_pre(self): + """Call parsers which must be called before tree structuring""" + ilines: Iterator[str] = iter(self._lines) + ilines = self._iparse_properties(ilines) + self._body_lines = list(ilines) + class OrgNode(OrgBaseNode): @@ -962,7 +1015,6 @@ def __init__(self, *args, **kwds) -> None: self._tags = cast(List[str], None) self._todo: Optional[str] = None self._priority = None - self._properties: Dict[str, PropertyValue] = {} self._scheduled = OrgDate(None) self._deadline = OrgDate(None) self._closed = OrgDate(None) @@ -1041,24 +1093,6 @@ def _iparse_timestamps(self, ilines: Iterator[str]) -> Iterator[str]: self._timestamps.extend(OrgDate.list_from_str(l)) yield l - def _iparse_properties(self, ilines: Iterator[str]) -> Iterator[str]: - self._properties = {} - in_property_field = False - for line in ilines: - if in_property_field: - if line.find(":END:") >= 0: - break - else: - (key, val) = parse_property(line) - if key is not None and val is not None: - self._properties.update({key: val}) - elif line.find(":PROPERTIES:") >= 0: - in_property_field = True - else: - yield line - for line in ilines: - yield line - def _iparse_repeated_tasks(self, ilines: Iterator[str]) -> Iterator[str]: self._repeated_tasks = [] for line in ilines: @@ -1169,37 +1203,6 @@ def todo(self) -> Optional[str]: """ return self._todo - def get_property(self, key, val=None) -> Optional[PropertyValue]: - """ - Return property named ``key`` if exists or ``val`` otherwise. - - :arg str key: - Key of property. - - :arg val: - Default value to return. - - """ - return self._properties.get(key, val) - - @property - def properties(self) -> Dict[str, PropertyValue]: - """ - Node properties as a dictionary. - - >>> from orgparse import loads - >>> root = loads(''' - ... * Node - ... :PROPERTIES: - ... :SomeProperty: value - ... :END: - ... ''') - >>> root.children[0].properties['SomeProperty'] - 'value' - - """ - return self._properties - @property def scheduled(self): """ @@ -1451,6 +1454,8 @@ def parse_lines(lines: Iterable[str], filename, env=None) -> OrgNode: nodelist.append(node) # parse headings (level, TODO, TAGs, and heading) nodelist[0]._index = 0 + # parse the root node + nodelist[0]._parse_pre() for (i, node) in enumerate(nodelist[1:], 1): # nodes except root node node._index = i node._parse_pre() diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index 732944d..a322cc9 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -169,3 +169,21 @@ def test_load_filelike() -> None: root = load(stream) assert len(root.children) == 2 assert root.env.filename == '' + + +def test_level_0_properties() -> None: + content = ''' +foo bar + +:PROPERTIES: +:PROP-FOO: Bar +:PROP-BAR: Bar bar +:END: + +* heading :h1: +** child :f2: + '''.strip() + root = loads(content) + assert root.get_property('PROP-FOO') == 'Bar' + assert root.get_property('PROP-BAR') == 'Bar bar' + assert root.get_property('PROP-INVALID') is None From dd94aad59cbc7ffd89e2488d1a5637850a597875 Mon Sep 17 00:00:00 2001 From: "Thomas Bruckmaier (Xubuntu Laptop)" Date: Fri, 3 Sep 2021 15:49:21 +0200 Subject: [PATCH 64/90] Added support for timestamps before first headline - moved get_timestamps(), datelist(), rangelist() to OrgBaseNode (so they are available also in OrgRootNode) and added timestamp parsing to OrgRootNode --- orgparse/node.py | 221 +++++++++++++++++++----------------- orgparse/tests/test_misc.py | 22 ++++ 2 files changed, 136 insertions(+), 107 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index 41ceaaf..b877f88 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -496,6 +496,7 @@ def __init__(self, env, index=None) -> None: self._lines: List[str] = [] self._properties: Dict[str, PropertyValue] = {} + self._timestamps: List[OrgDate] = [] # FIXME: use `index` argument to set index. (Currently it is # done externally in `parse_lines`.) @@ -932,6 +933,112 @@ def is_root(self): """ return False + def get_timestamps(self, active=False, inactive=False, + range=False, point=False): + """ + Return a list of timestamps in the body text. + + :type active: bool + :arg active: Include active type timestamps. + :type inactive: bool + :arg inactive: Include inactive type timestamps. + :type range: bool + :arg range: Include timestamps which has end date. + :type point: bool + :arg point: Include timestamps which has no end date. + + :rtype: list of :class:`orgparse.date.OrgDate` subclasses + + + Consider the following org node: + + >>> from orgparse import loads + >>> node = loads(''' + ... * Node + ... CLOSED: [2012-02-26 Sun 21:15] SCHEDULED: <2012-02-26 Sun> + ... CLOCK: [2012-02-26 Sun 21:10]--[2012-02-26 Sun 21:15] => 0:05 + ... Some inactive timestamp [2012-02-23 Thu] in body text. + ... Some active timestamp <2012-02-24 Fri> in body text. + ... Some inactive time range [2012-02-25 Sat]--[2012-02-27 Mon]. + ... Some active time range <2012-02-26 Sun>--<2012-02-28 Tue>. + ... ''').children[0] + + The default flags are all off, so it does not return anything. + + >>> node.get_timestamps() + [] + + You can fetch appropriate timestamps using keyword arguments. + + >>> node.get_timestamps(inactive=True, point=True) + [OrgDate((2012, 2, 23), None, False)] + >>> node.get_timestamps(active=True, point=True) + [OrgDate((2012, 2, 24))] + >>> node.get_timestamps(inactive=True, range=True) + [OrgDate((2012, 2, 25), (2012, 2, 27), False)] + >>> node.get_timestamps(active=True, range=True) + [OrgDate((2012, 2, 26), (2012, 2, 28))] + + This is more complex example. Only active timestamps, + regardless of range/point type. + + >>> node.get_timestamps(active=True, point=True, range=True) + [OrgDate((2012, 2, 24)), OrgDate((2012, 2, 26), (2012, 2, 28))] + + """ + return [ + ts for ts in self._timestamps if + (((active and ts.is_active()) or + (inactive and not ts.is_active())) and + ((range and ts.has_end()) or + (point and not ts.has_end())))] + + @property + def datelist(self): + """ + Alias of ``.get_timestamps(active=True, inactive=True, point=True)``. + + :rtype: list of :class:`orgparse.date.OrgDate` subclasses + + >>> from orgparse import loads + >>> root = loads(''' + ... * Node with point dates <2012-02-25 Sat> + ... CLOSED: [2012-02-25 Sat 21:15] + ... Some inactive timestamp [2012-02-26 Sun] in body text. + ... Some active timestamp <2012-02-27 Mon> in body text. + ... ''') + >>> root.children[0].datelist # doctest: +NORMALIZE_WHITESPACE + [OrgDate((2012, 2, 25)), + OrgDate((2012, 2, 26), None, False), + OrgDate((2012, 2, 27))] + + """ + return self.get_timestamps(active=True, inactive=True, point=True) + + @property + def rangelist(self): + """ + Alias of ``.get_timestamps(active=True, inactive=True, range=True)``. + + :rtype: list of :class:`orgparse.date.OrgDate` subclasses + + >>> from orgparse import loads + >>> root = loads(''' + ... * Node with range dates <2012-02-25 Sat>--<2012-02-28 Tue> + ... CLOCK: [2012-02-26 Sun 21:10]--[2012-02-26 Sun 21:15] => 0:05 + ... Some inactive time range [2012-02-25 Sat]--[2012-02-27 Mon]. + ... Some active time range <2012-02-26 Sun>--<2012-02-28 Tue>. + ... Some time interval <2012-02-27 Mon 11:23-12:10>. + ... ''') + >>> root.children[0].rangelist # doctest: +NORMALIZE_WHITESPACE + [OrgDate((2012, 2, 25), (2012, 2, 28)), + OrgDate((2012, 2, 25), (2012, 2, 27), False), + OrgDate((2012, 2, 26), (2012, 2, 28)), + OrgDate((2012, 2, 27, 11, 23, 0), (2012, 2, 27, 12, 10, 0))] + + """ + return self.get_timestamps(active=True, inactive=True, range=True) + def __unicode__(self): return unicode("\n").join(self._lines) @@ -995,8 +1102,15 @@ def _parse_pre(self): """Call parsers which must be called before tree structuring""" ilines: Iterator[str] = iter(self._lines) ilines = self._iparse_properties(ilines) + ilines = self._iparse_timestamps(ilines) self._body_lines = list(ilines) + def _iparse_timestamps(self, ilines: Iterator[str]) -> Iterator[str]: + self._timestamps = [] + for line in ilines: + self._timestamps.extend(OrgDate.list_from_str(line)) + yield line + class OrgNode(OrgBaseNode): @@ -1018,7 +1132,6 @@ def __init__(self, *args, **kwds) -> None: self._scheduled = OrgDate(None) self._deadline = OrgDate(None) self._closed = OrgDate(None) - self._timestamps: List[OrgDate] = [] self._clocklist: List[OrgDateClock] = [] self._body_lines: List[str] = [] self._repeated_tasks: List[OrgDateRepeatedTask] = [] @@ -1275,112 +1388,6 @@ def clock(self): """ return self._clocklist - def get_timestamps(self, active=False, inactive=False, - range=False, point=False): - """ - Return a list of timestamps in the body text. - - :type active: bool - :arg active: Include active type timestamps. - :type inactive: bool - :arg inactive: Include inactive type timestamps. - :type range: bool - :arg range: Include timestamps which has end date. - :type point: bool - :arg point: Include timestamps which has no end date. - - :rtype: list of :class:`orgparse.date.OrgDate` subclasses - - - Consider the following org node: - - >>> from orgparse import loads - >>> node = loads(''' - ... * Node - ... CLOSED: [2012-02-26 Sun 21:15] SCHEDULED: <2012-02-26 Sun> - ... CLOCK: [2012-02-26 Sun 21:10]--[2012-02-26 Sun 21:15] => 0:05 - ... Some inactive timestamp [2012-02-23 Thu] in body text. - ... Some active timestamp <2012-02-24 Fri> in body text. - ... Some inactive time range [2012-02-25 Sat]--[2012-02-27 Mon]. - ... Some active time range <2012-02-26 Sun>--<2012-02-28 Tue>. - ... ''').children[0] - - The default flags are all off, so it does not return anything. - - >>> node.get_timestamps() - [] - - You can fetch appropriate timestamps using keyword arguments. - - >>> node.get_timestamps(inactive=True, point=True) - [OrgDate((2012, 2, 23), None, False)] - >>> node.get_timestamps(active=True, point=True) - [OrgDate((2012, 2, 24))] - >>> node.get_timestamps(inactive=True, range=True) - [OrgDate((2012, 2, 25), (2012, 2, 27), False)] - >>> node.get_timestamps(active=True, range=True) - [OrgDate((2012, 2, 26), (2012, 2, 28))] - - This is more complex example. Only active timestamps, - regardless of range/point type. - - >>> node.get_timestamps(active=True, point=True, range=True) - [OrgDate((2012, 2, 24)), OrgDate((2012, 2, 26), (2012, 2, 28))] - - """ - return [ - ts for ts in self._timestamps if - (((active and ts.is_active()) or - (inactive and not ts.is_active())) and - ((range and ts.has_end()) or - (point and not ts.has_end())))] - - @property - def datelist(self): - """ - Alias of ``.get_timestamps(active=True, inactive=True, point=True)``. - - :rtype: list of :class:`orgparse.date.OrgDate` subclasses - - >>> from orgparse import loads - >>> root = loads(''' - ... * Node with point dates <2012-02-25 Sat> - ... CLOSED: [2012-02-25 Sat 21:15] - ... Some inactive timestamp [2012-02-26 Sun] in body text. - ... Some active timestamp <2012-02-27 Mon> in body text. - ... ''') - >>> root.children[0].datelist # doctest: +NORMALIZE_WHITESPACE - [OrgDate((2012, 2, 25)), - OrgDate((2012, 2, 26), None, False), - OrgDate((2012, 2, 27))] - - """ - return self.get_timestamps(active=True, inactive=True, point=True) - - @property - def rangelist(self): - """ - Alias of ``.get_timestamps(active=True, inactive=True, range=True)``. - - :rtype: list of :class:`orgparse.date.OrgDate` subclasses - - >>> from orgparse import loads - >>> root = loads(''' - ... * Node with range dates <2012-02-25 Sat>--<2012-02-28 Tue> - ... CLOCK: [2012-02-26 Sun 21:10]--[2012-02-26 Sun 21:15] => 0:05 - ... Some inactive time range [2012-02-25 Sat]--[2012-02-27 Mon]. - ... Some active time range <2012-02-26 Sun>--<2012-02-28 Tue>. - ... Some time interval <2012-02-27 Mon 11:23-12:10>. - ... ''') - >>> root.children[0].rangelist # doctest: +NORMALIZE_WHITESPACE - [OrgDate((2012, 2, 25), (2012, 2, 28)), - OrgDate((2012, 2, 25), (2012, 2, 27), False), - OrgDate((2012, 2, 26), (2012, 2, 28)), - OrgDate((2012, 2, 27, 11, 23, 0), (2012, 2, 27, 12, 10, 0))] - - """ - return self.get_timestamps(active=True, inactive=True, range=True) - def has_date(self): """ Return ``True`` if it has any kind of timestamp diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index a322cc9..eb57017 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -1,5 +1,6 @@ from .. import load, loads from ..node import OrgEnv +from orgparse.date import OrgDate def test_empty_heading() -> None: @@ -187,3 +188,24 @@ def test_level_0_properties() -> None: assert root.get_property('PROP-FOO') == 'Bar' assert root.get_property('PROP-BAR') == 'Bar bar' assert root.get_property('PROP-INVALID') is None + + +def test_level_0_timestamps() -> None: + content = ''' +foo bar + + - <2010-08-16 Mon> DateList + - <2010-08-07 Sat>--<2010-08-08 Sun> + - <2010-08-09 Mon 00:30>--<2010-08-10 Tue 13:20> RangeList + - <2019-08-10 Sat 16:30-17:30> TimeRange" + +* heading :h1: +** child :f2: + '''.strip() + root = loads(content) + assert root.datelist == [OrgDate((2010, 8, 16))] + assert root.rangelist == [ + OrgDate((2010, 8, 7), (2010, 8, 8)), + OrgDate((2010, 8, 9, 0, 30), (2010, 8, 10, 13, 20)), + OrgDate((2019, 8, 10, 16, 30, 0), (2019, 8, 10, 17, 30, 0)), + ] From 989d6c654c0322d7de9cd8af4c04ce34f13b6098 Mon Sep 17 00:00:00 2001 From: "Thomas Bruckmaier (Xubuntu Laptop)" Date: Fri, 3 Sep 2021 16:06:46 +0200 Subject: [PATCH 65/90] Improved tests for level-0 properties --- orgparse/tests/test_misc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index eb57017..a22d9e8 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -182,12 +182,17 @@ def test_level_0_properties() -> None: :END: * heading :h1: +:PROPERTIES: +:HEADING-PROP: foo +:END: ** child :f2: '''.strip() root = loads(content) assert root.get_property('PROP-FOO') == 'Bar' assert root.get_property('PROP-BAR') == 'Bar bar' assert root.get_property('PROP-INVALID') is None + assert root.get_property('HEADING-PROP') is None + assert root.children[0].get_property('HEADING-PROP') == 'foo' def test_level_0_timestamps() -> None: From 556a530ce9d6266273c3bf4ab7c00117bacf4ca6 Mon Sep 17 00:00:00 2001 From: Jariullah Date: Tue, 29 Jun 2021 19:37:03 +0200 Subject: [PATCH 66/90] Change + to * for repeated tasks to allow for entries at root --- orgparse/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orgparse/node.py b/orgparse/node.py index df079cd..6c5805c 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1076,7 +1076,7 @@ def _iparse_repeated_tasks(self, ilines: Iterator[str]) -> Iterator[str]: _repeated_tasks_re = re.compile( r''' - \s+ - \s+ + \s*- \s+ State \s+ "(?P [^"]+)" \s+ from \s+ "(?P [^"]+)" \s+ \[ (?P [^\]]+) \]''', From 0757d2376656385706f81ad27f5d90a650d4e552 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Sun, 31 Oct 2021 19:50:13 +0000 Subject: [PATCH 67/90] ci: update configs --- .ci/run | 2 +- .github/workflows/main.yml | 10 ++++++---- orgparse/node.py | 4 ---- pytest.ini | 1 + tox.ini | 3 ++- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/run b/.ci/run index a0a314a..45d3e82 100755 --- a/.ci/run +++ b/.ci/run @@ -10,7 +10,7 @@ if ! command -v sudo; then } fi -if ! [ -z "$CI" ]; then +if [ -n "${CI-}" ]; then # install OS specific stuff here if [[ "$OSTYPE" == "darwin"* ]]; then # macos diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cc58280..a31173a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,9 @@ jobs: strategy: matrix: platform: [ubuntu-latest, macos-latest] # TODO windows-latest?? - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + # seems like 3.6 isn't available on their osx image anymore + exclude: [{platform: macos-latest, python-version: '3.6'}] runs-on: ${{ matrix.platform }} @@ -29,7 +31,7 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -56,9 +58,9 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: '3.7' - uses: actions/checkout@v2 with: diff --git a/orgparse/node.py b/orgparse/node.py index 6c5805c..6fc0b81 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -1,10 +1,6 @@ import re import itertools from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict, Set, Sequence, Any -try: - from collections.abc import Sequence -except ImportError: - from collections import Sequence from .date import OrgDate, OrgDateClock, OrgDateRepeatedTask, parse_sdc from .inline import to_plain_text diff --git a/pytest.ini b/pytest.ini index 3b94dc4..28d084d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,6 +2,7 @@ # discover files that don't follow test_ naming. Useful to keep tests along with the source code python_files = *.py addopts = + # -rap to print tests summary even when they are successful -rap --verbose diff --git a/tox.ini b/tox.ini index b0324f6..31a0a55 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,8 @@ commands = [testenv:mypy] commands = pip install -e .[linting] - python -m mypy orgparse \ + python -m mypy --install-types --non-interactive \ + orgparse \ # txt report is a bit more convenient to view on CI --txt-report .coverage.mypy \ --html-report .coverage.mypy \ From 6783c5e9f6e8a66b4929a167fd0d2ac1982106f8 Mon Sep 17 00:00:00 2001 From: Romain Gehrig Date: Sun, 29 Aug 2021 13:12:54 +0200 Subject: [PATCH 68/90] Harmonize SDC date objects Nodes with only the heading returned OrgDate(None) for all SDC dates whereas nodes with multiple lines had the more precise object OrgDateScheduled(None)/... --- orgparse/node.py | 8 ++++---- orgparse/tests/data/01_attributes.org | 2 ++ orgparse/tests/data/01_attributes.py | 25 +++++++++++++++++++------ orgparse/tests/test_data.py | 1 + 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/orgparse/node.py b/orgparse/node.py index 6fc0b81..8496f03 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -2,7 +2,7 @@ import itertools from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict, Set, Sequence, Any -from .date import OrgDate, OrgDateClock, OrgDateRepeatedTask, parse_sdc +from .date import OrgDate, OrgDateClock, OrgDateRepeatedTask, parse_sdc, OrgDateScheduled, OrgDateDeadline, OrgDateClosed from .inline import to_plain_text from .extra import to_rich_text, Rich from .utils.py3compat import PY3, unicode @@ -959,9 +959,9 @@ def __init__(self, *args, **kwds) -> None: self._todo: Optional[str] = None self._priority = None self._properties: Dict[str, PropertyValue] = {} - self._scheduled = OrgDate(None) - self._deadline = OrgDate(None) - self._closed = OrgDate(None) + self._scheduled = OrgDateScheduled(None) + self._deadline = OrgDateDeadline(None) + self._closed = OrgDateClosed(None) self._timestamps: List[OrgDate] = [] self._clocklist: List[OrgDateClock] = [] self._body_lines: List[str] = [] diff --git a/orgparse/tests/data/01_attributes.org b/orgparse/tests/data/01_attributes.org index 8a047c5..99e202b 100644 --- a/orgparse/tests/data/01_attributes.org +++ b/orgparse/tests/data/01_attributes.org @@ -25,3 +25,5 @@ * range in deadline DEADLINE: <2019-09-06 Fri 10:00--11:20> body +* node with a second line but no date +body diff --git a/orgparse/tests/data/01_attributes.py b/orgparse/tests/data/01_attributes.py index 1ced627..d4555de 100644 --- a/orgparse/tests/data/01_attributes.py +++ b/orgparse/tests/data/01_attributes.py @@ -34,9 +34,9 @@ node2: Raw = dict( heading="A node without any attributed", priority=None, - scheduled=OrgDate(None), - deadline=OrgDate(None), - closed=OrgDate(None), + scheduled=OrgDateScheduled(None), + deadline=OrgDateDeadline(None), + closed=OrgDateClosed(None), clock=[], properties={}, datelist=[], @@ -47,9 +47,9 @@ node3: Raw = dict( heading="range in deadline", priority=None, - scheduled=OrgDate(None), + scheduled=OrgDateScheduled(None), deadline=OrgDateDeadline((2019, 9, 6, 10, 0), (2019, 9, 6, 11, 20)), - closed=OrgDate(None), + closed=OrgDateClosed(None), clock=[], properties={}, datelist=[], @@ -57,4 +57,17 @@ body=" body", ) -data = [node1, node2, node1, node3] +node4: Raw = dict( + heading="node with a second line but no date", + priority=None, + scheduled=OrgDateScheduled(None), + deadline=OrgDateDeadline(None), + closed=OrgDateClosed(None), + clock=[], + properties={}, + datelist=[], + rangelist=[], + body="body", +) + +data = [node1, node2, node1, node3, node4] diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index 5851513..286d6f0 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -65,6 +65,7 @@ def test_data(dataname): for key in kwds: val = value_from_data_key(node, key) assert kwds[key] == val, 'check value of {0}-th node of key "{1}" from "{2}".\n\nParsed:\n{3}\n\nReal:\n{4}'.format(i, key, dataname, val, kwds[key]) + assert type(kwds[key]) == type(val), 'check type of {0}-th node of key "{1}" from "{2}".\n\nParsed:\n{3}\n\nReal:\n{4}'.format(i, key, dataname, type(val), type(kwds[key])) assert root.env.filename == oname From d1bd16e46d5c8b40f5a241c16db708eb62de0c20 Mon Sep 17 00:00:00 2001 From: "Thomas Bruckmaier (Xubuntu Laptop)" Date: Fri, 3 Sep 2021 16:30:43 +0200 Subject: [PATCH 69/90] Fixed OrgDate._as_datetime, added test when a datetime object was passed, time information got lost (`datetime.datetime` is a child of `datetime.date`, so datetime object were wrongly converted) --- orgparse/date.py | 8 ++++++-- orgparse/tests/test_date.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 orgparse/tests/test_date.py diff --git a/orgparse/date.py b/orgparse/date.py index 9b1d8fa..c56d09d 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -333,8 +333,12 @@ def _datetime_in_range(self, date): return False @staticmethod - def _as_datetime(date): - if isinstance(date, datetime.date): + def _as_datetime(date) -> datetime.datetime: + """ + Convert the given date into datetime (if it already is, return it + unmodified + """ + if not isinstance(date, datetime.datetime): return datetime.datetime(*date.timetuple()[:3]) return date diff --git a/orgparse/tests/test_date.py b/orgparse/tests/test_date.py new file mode 100644 index 0000000..a28a75b --- /dev/null +++ b/orgparse/tests/test_date.py @@ -0,0 +1,10 @@ +from orgparse.date import OrgDate +import datetime + + +def test_date_as_datetime() -> None: + testdate = (2021, 9, 3) + testdatetime = (2021, 9, 3, 16, 19, 13) + + assert OrgDate._as_datetime(datetime.date(*testdate)) == datetime.datetime(*testdate, 0, 0, 0) + assert OrgDate._as_datetime(datetime.datetime(*testdatetime)) == datetime.datetime(*testdatetime) From 620ea35c08dce924e0fa547f5748f94befcc65c6 Mon Sep 17 00:00:00 2001 From: tbruckmaier Date: Sun, 31 Oct 2021 21:45:51 +0100 Subject: [PATCH 70/90] Added __str__ for OrgDate (#44) When converting OrgDate (or its descendants) to a string, use the correct org format (considering active status, ranges) Co-authored-by: karlicoss --- orgparse/date.py | 50 +++++++++++++++++++++++++++++++++++++ orgparse/tests/test_date.py | 31 ++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/orgparse/date.py b/orgparse/date.py index c56d09d..8fd1134 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -133,6 +133,28 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): return regex.format(prefix=prefix, ignore=ignore) +def date_time_format(date) -> str: + """ + Format a date or datetime in default org format + + @param date The date + + @return Formatted date(time) + """ + default_format_date = "%Y-%m-%d %a" + default_format_datetime = "%Y-%m-%d %a %H:%M" + is_datetime = isinstance(date, datetime.datetime) + + return date.strftime(default_format_datetime if is_datetime else default_format_date) + + +def is_same_day(date0, date1) -> bool: + """ + Check if two dates or datetimes are on the same day + """ + return (OrgDate._date_to_tuple(date0)[:3] == OrgDate._date_to_tuple(date1)[:3]) + + TIMESTAMP_NOBRACE_RE = re.compile( gene_timestamp_regex('nobrace', prefix=''), re.VERBOSE) @@ -154,6 +176,14 @@ class OrgDate(object): """ + """ + When formatting the date to string via __str__, and there is an end date on + the same day as the start date, allow formatting in the short syntax + <2021-09-03 Fri 16:01--17:30>? Otherwise the string represenation would be + <2021-09-03 Fri 16:01>--<2021-09-03 Fri 17:30> + """ + _allow_short_range = True + def __init__(self, start, end=None, active=None): """ Create :class:`OrgDate` object @@ -232,6 +262,24 @@ def __repr__(self): else: return '{0}({1!r}, {2!r}, {3!r})'.format(*args) + def __str__(self): + fence = ("<", ">") if self.is_active() else ("[", "]") + + start = date_time_format(self.start) + end = None + + if self.has_end(): + if self._allow_short_range and is_same_day(self.start, self.end): + start += "--%s" % self.end.strftime("%H:%M") + else: + end = date_time_format(self.end) + + ret = "%s%s%s" % (fence[0], start, fence[1]) + if end: + ret += "--%s%s%s" % (fence[0], end, fence[1]) + + return ret + def __nonzero__(self): return bool(self._start) @@ -498,6 +546,8 @@ class OrgDateClock(OrgDate): _active_default = False + _allow_short_range = False + def __init__(self, start, end, duration=None, active=None): """ Create OrgDateClock object diff --git a/orgparse/tests/test_date.py b/orgparse/tests/test_date.py index a28a75b..ef3689d 100644 --- a/orgparse/tests/test_date.py +++ b/orgparse/tests/test_date.py @@ -1,7 +1,36 @@ -from orgparse.date import OrgDate +from orgparse.date import OrgDate, OrgDateScheduled, OrgDateDeadline, OrgDateClock, OrgDateClosed import datetime +def test_date_as_string() -> None: + + testdate = datetime.date(2021, 9, 3) + testdate2 = datetime.date(2021, 9, 5) + testdatetime = datetime.datetime(2021, 9, 3, 16, 19, 13) + testdatetime2 = datetime.datetime(2021, 9, 3, 17, 0, 1) + testdatetime_nextday = datetime.datetime(2021, 9, 4, 0, 2, 1) + + assert str(OrgDate(testdate)) == "<2021-09-03 Fri>" + assert str(OrgDate(testdatetime)) == "<2021-09-03 Fri 16:19>" + assert str(OrgDate(testdate, active=False)) == "[2021-09-03 Fri]" + assert str(OrgDate(testdatetime, active=False)) == "[2021-09-03 Fri 16:19]" + + assert str(OrgDate(testdate, testdate2)) == "<2021-09-03 Fri>--<2021-09-05 Sun>" + assert str(OrgDate(testdate, testdate2)) == "<2021-09-03 Fri>--<2021-09-05 Sun>" + assert str(OrgDate(testdate, testdate2, active=False)) == "[2021-09-03 Fri]--[2021-09-05 Sun]" + assert str(OrgDate(testdate, testdate2, active=False)) == "[2021-09-03 Fri]--[2021-09-05 Sun]" + + assert str(OrgDateScheduled(testdate)) == "<2021-09-03 Fri>" + assert str(OrgDateScheduled(testdatetime)) == "<2021-09-03 Fri 16:19>" + assert str(OrgDateDeadline(testdate)) == "<2021-09-03 Fri>" + assert str(OrgDateDeadline(testdatetime)) == "<2021-09-03 Fri 16:19>" + assert str(OrgDateClosed(testdate)) == "[2021-09-03 Fri]" + assert str(OrgDateClosed(testdatetime)) == "[2021-09-03 Fri 16:19]" + + assert str(OrgDateClock(testdatetime, testdatetime2)) == "[2021-09-03 Fri 16:19]--[2021-09-03 Fri 17:00]" + assert str(OrgDateClock(testdatetime, testdatetime_nextday)) == "[2021-09-03 Fri 16:19]--[2021-09-04 Sat 00:02]" + + def test_date_as_datetime() -> None: testdate = (2021, 9, 3) testdatetime = (2021, 9, 3, 16, 19, 13) From ac78d8d36b3a851bab3d3becea899d65cd24123a Mon Sep 17 00:00:00 2001 From: Niclas Borlin Date: Fri, 15 Oct 2021 10:14:12 +0200 Subject: [PATCH 71/90] Ensure input file is closed within load() --- orgparse/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index b3a42d0..28342e9 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -131,13 +131,15 @@ def load(path: Union[str, Path, TextIO], env: Optional[OrgEnv]=None) -> OrgNode: """ orgfile: TextIO if isinstance(path, (str, Path)): - orgfile = codecs.open(str(path), encoding='utf8') + # Use 'with' to close the file inside this function. + with codecs.open(str(path), encoding='utf8') as orgfile: + lines = (l.rstrip('\n') for l in orgfile.readlines()) filename = str(path) else: orgfile = path + lines = (l.rstrip('\n') for l in orgfile.readlines()) filename = path.name if hasattr(path, 'name') else '' - return loadi((l.rstrip('\n') for l in orgfile.readlines()), - filename=filename, env=env) + return loadi(lines, filename=filename, env=env) def loads(string: str, filename: str='', env: Optional[OrgEnv]=None) -> OrgNode: From e79228fd0f2cedc6c6e21a3de5b73337d2749fbf Mon Sep 17 00:00:00 2001 From: Dr-Ops <96115647+Dr-Ops@users.noreply.github.com> Date: Tue, 14 Dec 2021 11:30:53 +0100 Subject: [PATCH 72/90] Hours are also allowed for repeats and warnings --- orgparse/date.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orgparse/date.py b/orgparse/date.py index 8fd1134..0d0b08e 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -114,13 +114,13 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): ({ignore}+?) (?P<{prefix}repeatpre> [\.\+]{{1,2}}) (?P<{prefix}repeatnum> \d+) - (?P<{prefix}repeatdwmy> [dwmy]) + (?P<{prefix}repeatdwmy> [hdwmy]) )? ( # optional warning ({ignore}+?) (?P<{prefix}warnpre> \-) (?P<{prefix}warnnum> \d+) - (?P<{prefix}warndwmy> [dwmy]) + (?P<{prefix}warndwmy> [hdwmy]) )? """ # http://www.pythonregex.com/ From 92c4a505fac3451d363857638f9a07a3fecd06cf Mon Sep 17 00:00:00 2001 From: "Thomas Bruckmaier (Xubuntu Laptop)" Date: Wed, 23 Feb 2022 16:26:07 +0100 Subject: [PATCH 73/90] Added support for open clock dates --- orgparse/date.py | 23 ++++++++++++++--------- orgparse/tests/data/04_logbook.org | 1 + orgparse/tests/data/04_logbook.py | 1 + orgparse/tests/test_date.py | 3 ++- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/orgparse/date.py b/orgparse/date.py index 0d0b08e..52a599d 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -548,7 +548,7 @@ class OrgDateClock(OrgDate): _allow_short_range = False - def __init__(self, start, end, duration=None, active=None): + def __init__(self, start, end=None, duration=None, active=None): """ Create OrgDateClock object """ @@ -600,20 +600,25 @@ def from_str(cls, line: str) -> 'OrgDateClock': match = cls._re.search(line) if not match: return cls(None, None) - groups = [int(d) for d in match.groups()] - ymdhm1 = groups[:5] - ymdhm2 = groups[5:10] - hm3 = groups[10:] + + ymdhm1 = [int(d) for d in match.groups()[:5]] + + # second part starting with "--", does not exist for open clock dates + has_end = bool(match.group(6)) + if has_end: + ymdhm2 = [int(d) for d in match.groups()[6:11]] + hm3 = [int(d) for d in match.groups()[11:]] + return cls( datetime.datetime(*ymdhm1), # type: ignore[arg-type] - datetime.datetime(*ymdhm2), # type: ignore[arg-type] - hm3[0] * 60 + hm3[1], + datetime.datetime(*ymdhm2) if has_end else None, # type: ignore[arg-type] + hm3[0] * 60 + hm3[1] if has_end else None, ) _re = re.compile( r'^(?!#).*CLOCK:\s+' - r'\[(\d+)\-(\d+)\-(\d+)[^\]\d]*(\d+)\:(\d+)\]--' - r'\[(\d+)\-(\d+)\-(\d+)[^\]\d]*(\d+)\:(\d+)\]\s+=>\s+(\d+)\:(\d+)' + r'\[(\d+)\-(\d+)\-(\d+)[^\]\d]*(\d+)\:(\d+)\]' + r'(--\[(\d+)\-(\d+)\-(\d+)[^\]\d]*(\d+)\:(\d+)\]\s+=>\s+(\d+)\:(\d+))?' ) diff --git a/orgparse/tests/data/04_logbook.org b/orgparse/tests/data/04_logbook.org index 9155470..e89ec26 100644 --- a/orgparse/tests/data/04_logbook.org +++ b/orgparse/tests/data/04_logbook.org @@ -1,5 +1,6 @@ * LOGBOOK drawer test :LOGBOOK: + CLOCK: [2012-10-26 Fri 16:01] CLOCK: [2012-10-26 Fri 14:50]--[2012-10-26 Fri 15:00] => 0:10 CLOCK: [2012-10-26 Fri 14:30]--[2012-10-26 Fri 14:40] => 0:10 CLOCK: [2012-10-26 Fri 14:10]--[2012-10-26 Fri 14:20] => 0:10 diff --git a/orgparse/tests/data/04_logbook.py b/orgparse/tests/data/04_logbook.py index 4c05a15..457c5fa 100644 --- a/orgparse/tests/data/04_logbook.py +++ b/orgparse/tests/data/04_logbook.py @@ -3,6 +3,7 @@ data = [dict( heading='LOGBOOK drawer test', clock=[ + OrgDateClock((2012, 10, 26, 16, 1)), OrgDateClock((2012, 10, 26, 14, 50), (2012, 10, 26, 15, 00)), OrgDateClock((2012, 10, 26, 14, 30), (2012, 10, 26, 14, 40)), OrgDateClock((2012, 10, 26, 14, 10), (2012, 10, 26, 14, 20)), diff --git a/orgparse/tests/test_date.py b/orgparse/tests/test_date.py index ef3689d..5763913 100644 --- a/orgparse/tests/test_date.py +++ b/orgparse/tests/test_date.py @@ -29,8 +29,9 @@ def test_date_as_string() -> None: assert str(OrgDateClock(testdatetime, testdatetime2)) == "[2021-09-03 Fri 16:19]--[2021-09-03 Fri 17:00]" assert str(OrgDateClock(testdatetime, testdatetime_nextday)) == "[2021-09-03 Fri 16:19]--[2021-09-04 Sat 00:02]" + assert str(OrgDateClock(testdatetime)) == "[2021-09-03 Fri 16:19]" + - def test_date_as_datetime() -> None: testdate = (2021, 9, 3) testdatetime = (2021, 9, 3, 16, 19, 13) From 36b31d8151e45ac6701822864816b48446d3dcad Mon Sep 17 00:00:00 2001 From: Johnson Date: Mon, 7 Mar 2022 04:33:12 +0800 Subject: [PATCH 74/90] Support parsing timestamp cookies --- orgparse/date.py | 43 ++++++++++++++++++++++++++++--------- orgparse/tests/test_date.py | 4 +++- orgparse/tests/test_misc.py | 27 +++++++++++++++++++++++ 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/orgparse/date.py b/orgparse/date.py index 52a599d..af9a9d8 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -184,7 +184,8 @@ class OrgDate(object): """ _allow_short_range = True - def __init__(self, start, end=None, active=None): + def __init__(self, start, end=None, active=None, repeater=None, + warning=None): """ Create :class:`OrgDate` object @@ -197,6 +198,10 @@ def __init__(self, start, end=None, active=None): :arg active: Active/inactive flag. None means using its default value, which may be different for different subclasses. + :type repeater: tuple or None + :arg repeater: Repeater interval. + :type warning: tuple or None + :arg warning: Deadline warning interval. >>> OrgDate(datetime.date(2012, 2, 10)) OrgDate((2012, 2, 10)) @@ -222,6 +227,9 @@ def __init__(self, start, end=None, active=None): self._start = self._to_date(start) self._end = self._to_date(end) self._active = self._active_default if active is None else active + # repeater and warning are tuples of (prefix, number, interval) + self._repeater = repeater + self._warning = warning @staticmethod def _to_date(date) -> DateIsh: @@ -254,13 +262,14 @@ def __repr__(self): self._date_to_tuple(self.start), self._date_to_tuple(self.end) if self.has_end() else None, None if self._active is self._active_default else self._active, + self._repeater, + self._warning, ] - if args[2] is None and args[3] is None: - return '{0}({1!r})'.format(*args) - elif args[3] is None: - return '{0}({1!r}, {2!r})'.format(*args) - else: - return '{0}({1!r}, {2!r}, {3!r})'.format(*args) + while args[-1] is None: + args.pop() + if len(args) > 3 and args[3] is None: + args[3] = self._active_default + return '{0}({1})'.format(args[0], ', '.join(map(repr, args[1:]))) def __str__(self): fence = ("<", ">") if self.is_active() else ("[", "]") @@ -274,6 +283,10 @@ def __str__(self): else: end = date_time_format(self.end) + if self._repeater: + start += " %s%d%s" % self._repeater + if self._warning: + start += " %s%d%s" % self._warning ret = "%s%s%s" % (fence[0], start, fence[1]) if end: ret += "--%s%s%s" % (fence[0], end, fence[1]) @@ -421,6 +434,7 @@ def list_from_str(cls, string: str) -> List['OrgDate']: >>> OrgDate.list_from_str("<2012-02-11 Sat 10:11--11:20>") [OrgDate((2012, 2, 11, 10, 11, 0), (2012, 2, 11, 11, 20, 0))] """ + cookie_suffix = ['pre', 'num', 'dwmy'] match = TIMESTAMP_RE.search(string) if match: rest = string[match.end():] @@ -433,6 +447,16 @@ def list_from_str(cls, string: str) -> List['OrgDate']: prefix = 'inactive_' active = False rangedash = '--[' + repeater: Optional[Tuple[str, int, str]] = None + warning: Optional[Tuple[str, int, str]] = None + if mdict[prefix + 'repeatpre'] is not None: + keys = [prefix + 'repeat' + suffix for suffix in cookie_suffix] + values = [mdict[k] for k in keys] + repeater = (values[0], int(values[1]), values[2]) + if mdict[prefix + 'warnpre'] is not None: + keys = [prefix + 'warn' + suffix for suffix in cookie_suffix] + values = [mdict[k] for k in keys] + warning = (values[0], int(values[1]), values[2]) has_rangedash = rest.startswith(rangedash) match2 = TIMESTAMP_RE.search(rest) if has_rangedash else None if has_rangedash and match2: @@ -442,12 +466,11 @@ def list_from_str(cls, string: str) -> List['OrgDate']: odate = cls( cls._datetuple_from_groupdict(mdict, prefix), cls._datetuple_from_groupdict(mdict2, prefix), - active=active) + active=active, repeater=repeater, warning=warning) else: odate = cls( *cls._daterange_from_groupdict(mdict, prefix), - active=active) - # FIXME: treat "repeater" and "warn" + active=active, repeater=repeater, warning=warning) return [odate] + cls.list_from_str(rest) else: return [] diff --git a/orgparse/tests/test_date.py b/orgparse/tests/test_date.py index 5763913..0f39575 100644 --- a/orgparse/tests/test_date.py +++ b/orgparse/tests/test_date.py @@ -17,8 +17,10 @@ def test_date_as_string() -> None: assert str(OrgDate(testdate, testdate2)) == "<2021-09-03 Fri>--<2021-09-05 Sun>" assert str(OrgDate(testdate, testdate2)) == "<2021-09-03 Fri>--<2021-09-05 Sun>" + assert str(OrgDate(testdatetime, testdatetime2)) == "<2021-09-03 Fri 16:19--17:00>" assert str(OrgDate(testdate, testdate2, active=False)) == "[2021-09-03 Fri]--[2021-09-05 Sun]" assert str(OrgDate(testdate, testdate2, active=False)) == "[2021-09-03 Fri]--[2021-09-05 Sun]" + assert str(OrgDate(testdatetime, testdatetime2, active=False)) == "[2021-09-03 Fri 16:19--17:00]" assert str(OrgDateScheduled(testdate)) == "<2021-09-03 Fri>" assert str(OrgDateScheduled(testdatetime)) == "<2021-09-03 Fri 16:19>" @@ -37,4 +39,4 @@ def test_date_as_datetime() -> None: testdatetime = (2021, 9, 3, 16, 19, 13) assert OrgDate._as_datetime(datetime.date(*testdate)) == datetime.datetime(*testdate, 0, 0, 0) - assert OrgDate._as_datetime(datetime.datetime(*testdatetime)) == datetime.datetime(*testdatetime) + assert OrgDate._as_datetime(datetime.datetime(*testdatetime)) == datetime.datetime(*testdatetime) \ No newline at end of file diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index a22d9e8..dcea8c3 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -214,3 +214,30 @@ def test_level_0_timestamps() -> None: OrgDate((2010, 8, 9, 0, 30), (2010, 8, 10, 13, 20)), OrgDate((2019, 8, 10, 16, 30, 0), (2019, 8, 10, 17, 30, 0)), ] + +def test_date_with_cookies() -> None: + testcases = [ + ('<2010-06-21 Mon +1y>', "OrgDate((2010, 6, 21), None, True, ('+', 1, 'y'))"), + ('<2005-10-01 Sat +1m>', "OrgDate((2005, 10, 1), None, True, ('+', 1, 'm'))"), + ('<2005-10-01 Sat +1m -3d>', "OrgDate((2005, 10, 1), None, True, ('+', 1, 'm'), ('-', 3, 'd'))"), + ('<2005-10-01 Sat -3d>', "OrgDate((2005, 10, 1), None, True, None, ('-', 3, 'd'))"), + ('<2008-02-10 Sun ++1w>', "OrgDate((2008, 2, 10), None, True, ('++', 1, 'w'))"), + ('<2008-02-08 Fri 20:00 ++1d>', "OrgDate((2008, 2, 8, 20, 0, 0), None, True, ('++', 1, 'd'))"), + ('<2019-04-05 Fri 08:00 .+1h>', "OrgDate((2019, 4, 5, 8, 0, 0), None, True, ('.+', 1, 'h'))"), + ('[2019-04-05 Fri 08:00 .+1h]', "OrgDate((2019, 4, 5, 8, 0, 0), None, False, ('.+', 1, 'h'))"), + ('<2007-05-16 Wed 12:30 +1w>', "OrgDate((2007, 5, 16, 12, 30, 0), None, True, ('+', 1, 'w'))"), + ] + for (input, expected) in testcases: + root = loads(input) + output = root[0].datelist[0] + assert str(output) == input + assert repr(output) == expected + testcases = [ + ('<2006-11-02 Thu 20:00-22:00 +1w>', "OrgDate((2006, 11, 2, 20, 0, 0), (2006, 11, 2, 22, 0, 0), True, ('+', 1, 'w'))"), + ('<2006-11-02 Thu 20:00--22:00 +1w>', "OrgDate((2006, 11, 2, 20, 0, 0), (2006, 11, 2, 22, 0, 0), True, ('+', 1, 'w'))"), + ] + for (input, expected) in testcases: + root = loads(input) + output = root[0].rangelist[0] + assert str(output) == "<2006-11-02 Thu 20:00--22:00 +1w>" + assert repr(output) == expected From 0041aad535836b5faf540205c8c68d5e42b52e19 Mon Sep 17 00:00:00 2001 From: "Kristijan \"Fremen\" Velkovski" Date: Tue, 6 Dec 2022 20:24:46 -0600 Subject: [PATCH 75/90] Test against Python 3.11, windows and update github actions. (#61) * Test against Python 3.11 * Update action versions. * fail-fast: false * drop python 3.6 from builds. --- .github/workflows/main.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a31173a..acd5204 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,11 +19,10 @@ env: jobs: build: strategy: + fail-fast: false matrix: - platform: [ubuntu-latest, macos-latest] # TODO windows-latest?? - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] - # seems like 3.6 isn't available on their osx image anymore - exclude: [{platform: macos-latest, python-version: '3.6'}] + platform: [ubuntu-latest, macos-latest, windows-latest] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] runs-on: ${{ matrix.platform }} @@ -31,11 +30,14 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/setup-python@v2 + - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }} + run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH + + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: recursive @@ -44,7 +46,7 @@ jobs: - run: .ci/run - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} path: .coverage.mypy/ @@ -58,11 +60,11 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: python-version: '3.7' - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: recursive From 28018a31ccec21f3ed95e5cd90ff4b2c924e6297 Mon Sep 17 00:00:00 2001 From: Johnson <20457146+j3soon@users.noreply.github.com> Date: Thu, 5 Jan 2023 04:21:21 +0800 Subject: [PATCH 76/90] Parse timestamp cookies for SCHEDULED and DEADLINE (#60) Parse timestamp cookies for SCHEDULED and DEADLINE --- orgparse/date.py | 15 ++++++- orgparse/tests/test_misc.py | 78 +++++++++++++++++++++++++++++++------ tox.ini | 4 +- 3 files changed, 83 insertions(+), 14 deletions(-) diff --git a/orgparse/date.py b/orgparse/date.py index af9a9d8..d000c97 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -527,7 +527,20 @@ def from_str(cls, string): end_dict.update(mdict) end_dict.update({'hour': end_hour, 'min': end_min}) end = cls._datetuple_from_groupdict(end_dict) - return cls(start, end, active=cls._active_default) + cookie_suffix = ['pre', 'num', 'dwmy'] + repeater: Optional[Tuple[str, int, str]] = None + warning: Optional[Tuple[str, int, str]] = None + prefix = '' + if mdict[prefix + 'repeatpre'] is not None: + keys = [prefix + 'repeat' + suffix for suffix in cookie_suffix] + values = [mdict[k] for k in keys] + repeater = (values[0], int(values[1]), values[2]) + if mdict[prefix + 'warnpre'] is not None: + keys = [prefix + 'warn' + suffix for suffix in cookie_suffix] + values = [mdict[k] for k in keys] + warning = (values[0], int(values[1]), values[2]) + return cls(start, end, active=cls._active_default, + repeater=repeater, warning=warning) else: return cls(None) diff --git a/orgparse/tests/test_misc.py b/orgparse/tests/test_misc.py index dcea8c3..4cd73e4 100644 --- a/orgparse/tests/test_misc.py +++ b/orgparse/tests/test_misc.py @@ -217,15 +217,24 @@ def test_level_0_timestamps() -> None: def test_date_with_cookies() -> None: testcases = [ - ('<2010-06-21 Mon +1y>', "OrgDate((2010, 6, 21), None, True, ('+', 1, 'y'))"), - ('<2005-10-01 Sat +1m>', "OrgDate((2005, 10, 1), None, True, ('+', 1, 'm'))"), - ('<2005-10-01 Sat +1m -3d>', "OrgDate((2005, 10, 1), None, True, ('+', 1, 'm'), ('-', 3, 'd'))"), - ('<2005-10-01 Sat -3d>', "OrgDate((2005, 10, 1), None, True, None, ('-', 3, 'd'))"), - ('<2008-02-10 Sun ++1w>', "OrgDate((2008, 2, 10), None, True, ('++', 1, 'w'))"), - ('<2008-02-08 Fri 20:00 ++1d>', "OrgDate((2008, 2, 8, 20, 0, 0), None, True, ('++', 1, 'd'))"), - ('<2019-04-05 Fri 08:00 .+1h>', "OrgDate((2019, 4, 5, 8, 0, 0), None, True, ('.+', 1, 'h'))"), - ('[2019-04-05 Fri 08:00 .+1h]', "OrgDate((2019, 4, 5, 8, 0, 0), None, False, ('.+', 1, 'h'))"), - ('<2007-05-16 Wed 12:30 +1w>', "OrgDate((2007, 5, 16, 12, 30, 0), None, True, ('+', 1, 'w'))"), + ('<2010-06-21 Mon +1y>', + "OrgDate((2010, 6, 21), None, True, ('+', 1, 'y'))"), + ('<2005-10-01 Sat +1m>', + "OrgDate((2005, 10, 1), None, True, ('+', 1, 'm'))"), + ('<2005-10-01 Sat +1m -3d>', + "OrgDate((2005, 10, 1), None, True, ('+', 1, 'm'), ('-', 3, 'd'))"), + ('<2005-10-01 Sat -3d>', + "OrgDate((2005, 10, 1), None, True, None, ('-', 3, 'd'))"), + ('<2008-02-10 Sun ++1w>', + "OrgDate((2008, 2, 10), None, True, ('++', 1, 'w'))"), + ('<2008-02-08 Fri 20:00 ++1d>', + "OrgDate((2008, 2, 8, 20, 0, 0), None, True, ('++', 1, 'd'))"), + ('<2019-04-05 Fri 08:00 .+1h>', + "OrgDate((2019, 4, 5, 8, 0, 0), None, True, ('.+', 1, 'h'))"), + ('[2019-04-05 Fri 08:00 .+1h]', + "OrgDate((2019, 4, 5, 8, 0, 0), None, False, ('.+', 1, 'h'))"), + ('<2007-05-16 Wed 12:30 +1w>', + "OrgDate((2007, 5, 16, 12, 30, 0), None, True, ('+', 1, 'w'))"), ] for (input, expected) in testcases: root = loads(input) @@ -233,11 +242,58 @@ def test_date_with_cookies() -> None: assert str(output) == input assert repr(output) == expected testcases = [ - ('<2006-11-02 Thu 20:00-22:00 +1w>', "OrgDate((2006, 11, 2, 20, 0, 0), (2006, 11, 2, 22, 0, 0), True, ('+', 1, 'w'))"), - ('<2006-11-02 Thu 20:00--22:00 +1w>', "OrgDate((2006, 11, 2, 20, 0, 0), (2006, 11, 2, 22, 0, 0), True, ('+', 1, 'w'))"), + ('<2006-11-02 Thu 20:00-22:00 +1w>', + "OrgDate((2006, 11, 2, 20, 0, 0), (2006, 11, 2, 22, 0, 0), True, ('+', 1, 'w'))"), + ('<2006-11-02 Thu 20:00--22:00 +1w>', + "OrgDate((2006, 11, 2, 20, 0, 0), (2006, 11, 2, 22, 0, 0), True, ('+', 1, 'w'))"), ] for (input, expected) in testcases: root = loads(input) output = root[0].rangelist[0] assert str(output) == "<2006-11-02 Thu 20:00--22:00 +1w>" assert repr(output) == expected + # DEADLINE and SCHEDULED + testcases2 = [ + ('* TODO Pay the rent\nDEADLINE: <2005-10-01 Sat +1m>', + "<2005-10-01 Sat +1m>", + "OrgDateDeadline((2005, 10, 1), None, True, ('+', 1, 'm'))"), + ('* TODO Pay the rent\nDEADLINE: <2005-10-01 Sat +1m -3d>', + "<2005-10-01 Sat +1m -3d>", + "OrgDateDeadline((2005, 10, 1), None, True, ('+', 1, 'm'), ('-', 3, 'd'))"), + ('* TODO Pay the rent\nDEADLINE: <2005-10-01 Sat -3d>', + "<2005-10-01 Sat -3d>", + "OrgDateDeadline((2005, 10, 1), None, True, None, ('-', 3, 'd'))"), + ('* TODO Pay the rent\nDEADLINE: <2005-10-01 Sat ++1m>', + "<2005-10-01 Sat ++1m>", + "OrgDateDeadline((2005, 10, 1), None, True, ('++', 1, 'm'))"), + ('* TODO Pay the rent\nDEADLINE: <2005-10-01 Sat .+1m>', + "<2005-10-01 Sat .+1m>", + "OrgDateDeadline((2005, 10, 1), None, True, ('.+', 1, 'm'))"), + ] + for (input, expected_str, expected_repr) in testcases2: + root = loads(input) + output = root[1].deadline + assert str(output) == expected_str + assert repr(output) == expected_repr + testcases2 = [ + ('* TODO Pay the rent\nSCHEDULED: <2005-10-01 Sat +1m>', + "<2005-10-01 Sat +1m>", + "OrgDateScheduled((2005, 10, 1), None, True, ('+', 1, 'm'))"), + ('* TODO Pay the rent\nSCHEDULED: <2005-10-01 Sat +1m -3d>', + "<2005-10-01 Sat +1m -3d>", + "OrgDateScheduled((2005, 10, 1), None, True, ('+', 1, 'm'), ('-', 3, 'd'))"), + ('* TODO Pay the rent\nSCHEDULED: <2005-10-01 Sat -3d>', + "<2005-10-01 Sat -3d>", + "OrgDateScheduled((2005, 10, 1), None, True, None, ('-', 3, 'd'))"), + ('* TODO Pay the rent\nSCHEDULED: <2005-10-01 Sat ++1m>', + "<2005-10-01 Sat ++1m>", + "OrgDateScheduled((2005, 10, 1), None, True, ('++', 1, 'm'))"), + ('* TODO Pay the rent\nSCHEDULED: <2005-10-01 Sat .+1m>', + "<2005-10-01 Sat .+1m>", + "OrgDateScheduled((2005, 10, 1), None, True, ('.+', 1, 'm'))"), + ] + for (input, expected_str, expected_repr) in testcases2: + root = loads(input) + output = root[1].scheduled + assert str(output) == expected_str + assert repr(output) == expected_repr diff --git a/tox.ini b/tox.ini index 31a0a55..3adf49d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,10 @@ [tox] -minversion = 3.5 +minversion = 3.7 # relies on the correct version of Python installed envlist = tests,mypy [testenv] -passenv = CI CI_* +passenv = CI,CI_* [testenv:tests] commands = From e92a30109d91dcab6ab4a6ab906ce923a51830ad Mon Sep 17 00:00:00 2001 From: buhtz Date: Wed, 4 Jan 2023 21:51:33 +0100 Subject: [PATCH 77/90] refactor load method to allow for easier mocking with pyfakefs (#57) --- orgparse/__init__.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/orgparse/__init__.py b/orgparse/__init__.py index 28342e9..e3f45cb 100644 --- a/orgparse/__init__.py +++ b/orgparse/__init__.py @@ -106,20 +106,19 @@ """ # [[[end]]] -import codecs +from io import IOBase from pathlib import Path from typing import Iterable, Union, Optional, TextIO -from .node import parse_lines, OrgEnv, OrgNode # todo basenode?? -from .utils.py3compat import basestring +from .node import parse_lines, OrgEnv, OrgNode # todo basenode?? __author__ = 'Takafumi Arakaki, Dmitrii Gerasimov' __license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] -def load(path: Union[str, Path, TextIO], env: Optional[OrgEnv]=None) -> OrgNode: +def load(path: Union[str, Path, TextIO], env: Optional[OrgEnv] = None) -> OrgNode: """ Load org-mode document from a file. @@ -129,17 +128,24 @@ def load(path: Union[str, Path, TextIO], env: Optional[OrgEnv]=None) -> OrgNode: :rtype: :class:`orgparse.node.OrgRootNode` """ - orgfile: TextIO - if isinstance(path, (str, Path)): - # Use 'with' to close the file inside this function. - with codecs.open(str(path), encoding='utf8') as orgfile: - lines = (l.rstrip('\n') for l in orgfile.readlines()) - filename = str(path) - else: - orgfile = path - lines = (l.rstrip('\n') for l in orgfile.readlines()) - filename = path.name if hasattr(path, 'name') else '' - return loadi(lines, filename=filename, env=env) + # Make sure it is a Path object. + if isinstance(path, str): + path = Path(path) + + # if it is a Path + if isinstance(path, Path): + # open that Path + with path.open('r', encoding='utf8') as orgfile: + # try again loading + return load(orgfile, env) + + # We assume it is a file-like object (e.g. io.StringIO) + all_lines = (line.rstrip('\n') for line in path) + + # get the filename + filename = path.name if hasattr(path, 'name') else '' + + return loadi(all_lines, filename=filename, env=env) def loads(string: str, filename: str='', env: Optional[OrgEnv]=None) -> OrgNode: From a720b83ac7eeefc6419883455135a69ec00dd0f3 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 2 Oct 2023 15:23:21 +0100 Subject: [PATCH 78/90] general: get rid of py3compat stuff, now everything should be on python3 --- orgparse/node.py | 11 ++--------- orgparse/tests/test_data.py | 12 +++--------- orgparse/utils/__init__.py | 0 orgparse/utils/_py3compat.py | 9 --------- orgparse/utils/py3compat.py | 14 -------------- 5 files changed, 5 insertions(+), 41 deletions(-) delete mode 100644 orgparse/utils/__init__.py delete mode 100644 orgparse/utils/_py3compat.py delete mode 100644 orgparse/utils/py3compat.py diff --git a/orgparse/node.py b/orgparse/node.py index 4c02ba9..154a3e8 100644 --- a/orgparse/node.py +++ b/orgparse/node.py @@ -5,7 +5,6 @@ from .date import OrgDate, OrgDateClock, OrgDateRepeatedTask, parse_sdc, OrgDateScheduled, OrgDateDeadline, OrgDateClosed from .inline import to_plain_text from .extra import to_rich_text, Rich -from .utils.py3compat import PY3, unicode def lines_to_chunks(lines: Iterable[str]) -> Iterable[List[str]]: @@ -1035,14 +1034,8 @@ def rangelist(self): """ return self.get_timestamps(active=True, inactive=True, range=True) - def __unicode__(self): - return unicode("\n").join(self._lines) - - if PY3: - __str__ = __unicode__ - else: - def __str__(self): - return unicode(self).encode('utf-8') + def __str__(self) -> str: + return "\n".join(self._lines) # todo hmm, not sure if it really belongs here and not to OrgRootNode? def get_file_property_list(self, property): diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index 286d6f0..b822474 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -1,10 +1,9 @@ import os from glob import glob +from pathlib import Path import pickle from .. import load, loads -from ..utils.py3compat import execfile, PY3 - import pytest # type: ignore DATADIR = os.path.join(os.path.dirname(__file__), 'data') @@ -12,8 +11,8 @@ def load_data(path): """Load data from python file""" - ns = {} # type: ignore - execfile(path, ns) + ns = {} # type: ignore + exec(Path(path).read_text(), ns) return ns['data'] @@ -52,11 +51,6 @@ def test_data(dataname): """ Compare parsed data from 'data/*.org' and its correct answer 'data/*.py' """ - if dataname == '05_tags': - if not PY3: - # python2 is end of life, so not worth fixing properly - pytest.skip('Ignoring test involving unicode') - oname = data_path(dataname, "org") data = load_data(data_path(dataname, "py")) root = load(oname) diff --git a/orgparse/utils/__init__.py b/orgparse/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/orgparse/utils/_py3compat.py b/orgparse/utils/_py3compat.py deleted file mode 100644 index 820e310..0000000 --- a/orgparse/utils/_py3compat.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Python 3 compatibility code which is loaded only when from Python 3. -""" - - -def execfile(filename, *args): - return exec( - compile(open(filename).read(), filename, 'exec'), - *args) diff --git a/orgparse/utils/py3compat.py b/orgparse/utils/py3compat.py deleted file mode 100644 index 2d5fa6a..0000000 --- a/orgparse/utils/py3compat.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -PY3 = (sys.version_info[0] >= 3) - -if PY3: - basestring = unicode = str -else: - unicode = unicode - basestring = basestring - -if PY3: - from ._py3compat import execfile -else: - execfile = execfile From 0476e8d8aa1d7109ac77410a88d9377316f9c630 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 2 Oct 2023 15:39:49 +0100 Subject: [PATCH 79/90] general: update mypy config; fix some issues --- mypy.ini | 7 ++++++- orgparse/date.py | 14 +++++++++++--- orgparse/extra.py | 8 ++++---- orgparse/tests/data/00_simple.py | 9 ++++++--- orgparse/tests/data/02_tree_struct.py | 7 +++++-- orgparse/tests/test_data.py | 5 +++-- orgparse/tests/test_hugedata.py | 7 ++----- orgparse/tests/test_rich.py | 2 +- 8 files changed, 38 insertions(+), 21 deletions(-) diff --git a/mypy.ini b/mypy.ini index 0f8386f..5a21a85 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,9 +1,14 @@ [mypy] +namespace_packages = True pretty = True show_error_context = True show_error_codes = True +show_column_numbers = True +show_error_end = True +warn_unused_ignores = True check_untyped_defs = True -namespace_packages = True +enable_error_code = possibly-undefined +strict_equality = True # an example of suppressing # [mypy-my.config.repos.pdfannots.pdfannots] diff --git a/orgparse/date.py b/orgparse/date.py index d000c97..dd407b7 100644 --- a/orgparse/date.py +++ b/orgparse/date.py @@ -641,14 +641,22 @@ def from_str(cls, line: str) -> 'OrgDateClock': # second part starting with "--", does not exist for open clock dates has_end = bool(match.group(6)) + ymdhm2_dt: Optional[datetime.datetime] + len_min: Optional[int] if has_end: ymdhm2 = [int(d) for d in match.groups()[6:11]] hm3 = [int(d) for d in match.groups()[11:]] + ymdhm2_dt = datetime.datetime(*ymdhm2) # type: ignore[arg-type] + len_min = hm3[0] * 60 + hm3[1] + else: + ymdhm2_dt = None + len_min = None + return cls( - datetime.datetime(*ymdhm1), # type: ignore[arg-type] - datetime.datetime(*ymdhm2) if has_end else None, # type: ignore[arg-type] - hm3[0] * 60 + hm3[1] if has_end else None, + datetime.datetime(*ymdhm1), # type: ignore[arg-type] + ymdhm2_dt, + len_min, ) _re = re.compile( diff --git a/orgparse/extra.py b/orgparse/extra.py index f9aab7a..cd51aba 100644 --- a/orgparse/extra.py +++ b/orgparse/extra.py @@ -1,5 +1,5 @@ import re -from typing import List, Sequence, Dict, Iterator, Iterable, Union, Optional +from typing import List, Sequence, Dict, Iterator, Iterable, Union, Optional, Type RE_TABLE_SEPARATOR = re.compile(r'\s*\|(\-+\+)*\-+\|') @@ -90,7 +90,7 @@ def to_rich_text(text: str) -> Iterator[Rich]: ''' lines = text.splitlines(keepends=True) group: List[str] = [] - last = Gap + last: Type[Rich] = Gap def emit() -> Rich: nonlocal group, last if last is Gap: @@ -106,11 +106,11 @@ def emit() -> Rich: if RE_TABLE_ROW.match(line) or RE_TABLE_SEPARATOR.match(line): cur = Table else: - cur = Gap # type: ignore + cur = Gap # type: ignore if cur is not last: if len(group) > 0: yield emit() - last = cur # type: ignore + last = cur group.append(line) if len(group) > 0: yield emit() diff --git a/orgparse/tests/data/00_simple.py b/orgparse/tests/data/00_simple.py index 0e68f13..c0b23d1 100644 --- a/orgparse/tests/data/00_simple.py +++ b/orgparse/tests/data/00_simple.py @@ -1,4 +1,7 @@ -def nodedict(i, level, todo=None, shallow_tags=set([]), tags=set([])): +from typing import Any, Dict, Set + + +def nodedict(i, level, todo=None, shallow_tags=set([]), tags=set([])) -> Dict[str, Any]: return dict( heading="Heading {0}".format(i), level=level, @@ -8,12 +11,12 @@ def nodedict(i, level, todo=None, shallow_tags=set([]), tags=set([])): ) -def tags(nums): +def tags(nums) -> Set[str]: return set(map('TAG{0}'.format, nums)) data = [ - nodedict(i, *vals) for (i, vals) in enumerate([ # type: ignore[misc] + nodedict(i, *vals) for (i, vals) in enumerate([ # type: ignore[misc] [1, 'TODO1', tags([1]) , tags(range(1, 2))], [2, 'TODO2', tags([2]) , tags(range(1, 3))], [3, 'TODO3', tags([3]) , tags(range(1, 4))], diff --git a/orgparse/tests/data/02_tree_struct.py b/orgparse/tests/data/02_tree_struct.py index c93aa56..80a8e77 100644 --- a/orgparse/tests/data/02_tree_struct.py +++ b/orgparse/tests/data/02_tree_struct.py @@ -1,11 +1,14 @@ -def nodedict(parent, children=[], previous=None, next=None): +from typing import Any, Dict + + +def nodedict(parent, children=[], previous=None, next=None) -> Dict[str, Any]: return dict(parent_heading=parent, children_heading=children, previous_same_level_heading=previous, next_same_level_heading=next) -data = [nodedict(*args) for args in [ # type: ignore[misc] +data = [nodedict(*args) for args in [ # G0 (None, [], None, 'G1-H1'), # G1 diff --git a/orgparse/tests/test_data.py b/orgparse/tests/test_data.py index b822474..75e75e4 100644 --- a/orgparse/tests/test_data.py +++ b/orgparse/tests/test_data.py @@ -1,10 +1,11 @@ -import os from glob import glob +import os from pathlib import Path import pickle from .. import load, loads -import pytest # type: ignore + +import pytest DATADIR = os.path.join(os.path.dirname(__file__), 'data') diff --git a/orgparse/tests/test_hugedata.py b/orgparse/tests/test_hugedata.py index b768ae8..f7248ca 100644 --- a/orgparse/tests/test_hugedata.py +++ b/orgparse/tests/test_hugedata.py @@ -1,7 +1,4 @@ -try: - import cPickle as pickle # type: ignore[import] -except ImportError: - import pickle # type: ignore[no-redef] +import pickle from .. import loadi @@ -24,7 +21,7 @@ def num_generate_org_lines(num_top_nodes, depth=3, nodes_per_level=1): nodes_per_level, depth - 1, nodes_per_level)) -def test_picklable(): +def test_picklable() -> None: num = 1000 depth = 3 nodes_per_level = 1 diff --git a/orgparse/tests/test_rich.py b/orgparse/tests/test_rich.py index 68fab40..7fb911b 100644 --- a/orgparse/tests/test_rich.py +++ b/orgparse/tests/test_rich.py @@ -4,7 +4,7 @@ from .. import load, loads from ..extra import Table -import pytest # type: ignore +import pytest def test_table() -> None: From db533f658fb9f5542295c96ea9c6e5c8c63cbdcf Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 2 Oct 2023 16:01:36 +0100 Subject: [PATCH 80/90] general: migrate to src/ package structure, update CI configs --- .ci/release | 4 +- .ci/run | 32 ++++++++++++---- .github/workflows/main.yml | 29 ++++++++------ Makefile | 2 +- conftest.py | 38 +++++++++++++++++++ pytest.ini | 3 ++ setup.py | 15 ++++---- {orgparse => src/orgparse}/__init__.py | 2 - {orgparse => src/orgparse}/date.py | 0 {orgparse => src/orgparse}/extra.py | 0 {orgparse => src/orgparse}/inline.py | 0 {orgparse => src/orgparse}/node.py | 0 {orgparse => src/orgparse}/py.typed | 0 {orgparse => src/orgparse}/tests/__init__.py | 0 .../orgparse}/tests/data/00_simple.org | 0 .../orgparse}/tests/data/00_simple.py | 0 .../orgparse}/tests/data/01_attributes.org | 0 .../orgparse}/tests/data/01_attributes.py | 0 .../orgparse}/tests/data/02_tree_struct.org | 0 .../orgparse}/tests/data/02_tree_struct.py | 0 .../tests/data/03_repeated_tasks.org | 0 .../orgparse}/tests/data/03_repeated_tasks.py | 0 .../orgparse}/tests/data/04_logbook.org | 0 .../orgparse}/tests/data/04_logbook.py | 0 .../orgparse}/tests/data/05_tags.org | 0 .../orgparse}/tests/data/05_tags.py | 0 .../orgparse}/tests/data/__init__.py | 0 {orgparse => src/orgparse}/tests/test_data.py | 3 +- {orgparse => src/orgparse}/tests/test_date.py | 0 .../orgparse}/tests/test_hugedata.py | 0 {orgparse => src/orgparse}/tests/test_misc.py | 0 {orgparse => src/orgparse}/tests/test_rich.py | 0 tox.ini | 38 ++++++++++++++----- 33 files changed, 124 insertions(+), 42 deletions(-) create mode 100644 conftest.py rename {orgparse => src/orgparse}/__init__.py (98%) rename {orgparse => src/orgparse}/date.py (100%) rename {orgparse => src/orgparse}/extra.py (100%) rename {orgparse => src/orgparse}/inline.py (100%) rename {orgparse => src/orgparse}/node.py (100%) rename {orgparse => src/orgparse}/py.typed (100%) rename {orgparse => src/orgparse}/tests/__init__.py (100%) rename {orgparse => src/orgparse}/tests/data/00_simple.org (100%) rename {orgparse => src/orgparse}/tests/data/00_simple.py (100%) rename {orgparse => src/orgparse}/tests/data/01_attributes.org (100%) rename {orgparse => src/orgparse}/tests/data/01_attributes.py (100%) rename {orgparse => src/orgparse}/tests/data/02_tree_struct.org (100%) rename {orgparse => src/orgparse}/tests/data/02_tree_struct.py (100%) rename {orgparse => src/orgparse}/tests/data/03_repeated_tasks.org (100%) rename {orgparse => src/orgparse}/tests/data/03_repeated_tasks.py (100%) rename {orgparse => src/orgparse}/tests/data/04_logbook.org (100%) rename {orgparse => src/orgparse}/tests/data/04_logbook.py (100%) rename {orgparse => src/orgparse}/tests/data/05_tags.org (100%) rename {orgparse => src/orgparse}/tests/data/05_tags.py (100%) rename {orgparse => src/orgparse}/tests/data/__init__.py (100%) rename {orgparse => src/orgparse}/tests/test_data.py (96%) rename {orgparse => src/orgparse}/tests/test_date.py (100%) rename {orgparse => src/orgparse}/tests/test_hugedata.py (100%) rename {orgparse => src/orgparse}/tests/test_misc.py (100%) rename {orgparse => src/orgparse}/tests/test_rich.py (100%) diff --git a/.ci/release b/.ci/release index 0ec687f..95e41f4 100755 --- a/.ci/release +++ b/.ci/release @@ -21,7 +21,7 @@ import shutil is_ci = os.environ.get('CI') is not None -def main(): +def main() -> None: import argparse p = argparse.ArgumentParser() p.add_argument('--test', action='store_true', help='use test pypi') @@ -29,7 +29,7 @@ def main(): extra = [] if args.test: - extra.extend(['--repository-url', 'https://test.pypi.org/legacy/']) + extra.extend(['--repository', 'testpypi']) root = Path(__file__).absolute().parent.parent os.chdir(root) # just in case diff --git a/.ci/run b/.ci/run index 45d3e82..b2c184d 100755 --- a/.ci/run +++ b/.ci/run @@ -1,10 +1,11 @@ -#!/bin/bash -eu +#!/bin/bash +set -eu cd "$(dirname "$0")" -cd .. +cd .. # git root if ! command -v sudo; then - # CI or Docker sometimes don't have it, so useful to have a dummy + # CI or Docker sometimes doesn't have it, so useful to have a dummy function sudo { "$@" } @@ -12,13 +13,28 @@ fi if [ -n "${CI-}" ]; then # install OS specific stuff here - if [[ "$OSTYPE" == "darwin"* ]]; then + case "$OSTYPE" in + darwin*) # macos : - else + ;; + cygwin* | msys* | win*) + # windows : - fi + ;; + *) + # must be linux? + : + ;; + esac +fi + + +PY_BIN="python3" +# some systems might have python pointing to python3 +if ! command -v python3 &> /dev/null; then + PY_BIN="python" fi -pip3 install --user tox -tox +"$PY_BIN" -m pip install --user tox +"$PY_BIN" -m tox --parallel --parallel-live "$@" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index acd5204..89338bd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,11 +10,14 @@ on: pull_request: # needed to trigger on others' PRs # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them". workflow_dispatch: # needed to trigger workflows manually - # todo cron? + # todo cron? + inputs: + debug_enabled: + type: boolean + description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + required: false + default: false -env: - # useful for scripts & sometimes tests to know - CI: true jobs: build: @@ -22,15 +25,17 @@ jobs: fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12.0-rc.3'] + # vvv just an example of excluding stuff from matrix + # exclude: [{platform: macos-latest, python-version: '3.6'}] runs-on: ${{ matrix.platform }} steps: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }} + # hmm somehow only seems necessary for 3.11 on osx?? run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH - uses: actions/setup-python@v4 @@ -41,12 +46,14 @@ jobs: with: submodules: recursive - # uncomment for SSH debugging - # - uses: mxschmitt/action-tmate@v3 + - uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - run: .ci/run + # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd... + - run: bash .ci/run - - uses: actions/upload-artifact@v3 + - if: matrix.platform == 'ubuntu-latest' # no need to compute coverage for other platforms + uses: actions/upload-artifact@v3 with: name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} path: .coverage.mypy/ @@ -62,7 +69,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: '3.7' + python-version: '3.8' - uses: actions/checkout@v3 with: diff --git a/Makefile b/Makefile index c22f6de..d31c4ea 100644 --- a/Makefile +++ b/Makefile @@ -11,4 +11,4 @@ doc: cog ## Update files using cog.py cog: orgparse/__init__.py orgparse/__init__.py: README.rst - cd orgparse && cog.py -r __init__.py + cd src/orgparse && cog.py -r __init__.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..466da24 --- /dev/null +++ b/conftest.py @@ -0,0 +1,38 @@ +# this is a hack to monkey patch pytest so it handles tests inside namespace packages without __init__.py properly +# without it, pytest can't discover the package root for some reason +# also see https://github.com/karlicoss/pytest_namespace_pkgs for more + +import pathlib +from typing import Optional + +import _pytest.main +import _pytest.pathlib + +# we consider all dirs in repo/ to be namespace packages +root_dir = pathlib.Path(__file__).absolute().parent.resolve() / 'src' +assert root_dir.exists(), root_dir + +# TODO assert it contains package name?? maybe get it via setuptools.. + +namespace_pkg_dirs = [str(d) for d in root_dir.iterdir() if d.is_dir()] + +# resolve_package_path is called from _pytest.pathlib.import_path +# takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem +resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path +def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]: + result = path # search from the test file upwards + for parent in result.parents: + if str(parent) in namespace_pkg_dirs: + return parent + raise RuntimeError("Couldn't determine path for ", path) +_pytest.pathlib.resolve_package_path = resolve_package_path + + +# without patching, the orig function returns just a package name for some reason +# (I think it's used as a sort of fallback) +# so we need to point it at the absolute path properly +# not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure.. +search_pypath_orig = _pytest.main.search_pypath +def search_pypath(module_name: str) -> str: + return str(root_dir) +_pytest.main.search_pypath = search_pypath diff --git a/pytest.ini b/pytest.ini index 28d084d..20c3704 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,3 +8,6 @@ addopts = # otherwise it won't discover doctests --doctest-modules + + # show all test durations (unless they are too short) + --durations=0 diff --git a/setup.py b/setup.py index 5f02a7d..5d500b1 100644 --- a/setup.py +++ b/setup.py @@ -4,10 +4,9 @@ def main(): - pkg = 'orgparse' - subpkgs = find_namespace_packages('.', include=(pkg + '.*',)) + pkgs = find_namespace_packages('src') + pkg = min(pkgs) - import orgparse setup( name=pkg, use_scm_version={ @@ -18,22 +17,24 @@ def main(): zip_safe=False, - packages=[pkg, *subpkgs], + packages=pkgs, + package_dir={'': 'src'}, package_data={ pkg: ['py.typed'], # todo need the rest as well?? 'orgparse.tests.data': ['*.org'], }, - author=orgparse.__author__, + author='Takafumi Arakaki, Dmitrii Gerasimov', author_email='aka.tkf@gmail.com', maintainer='Dima Gerasimov (@karlicoss)', maintainer_email='karlicoss@gmail.com', url='https://github.com/karlicoss/orgparse', - license=orgparse.__license__, + license='BSD License', description='orgparse - Emacs org-mode parser in Python', - long_description=orgparse.__doc__, + # TODO add it back later, perhaps via ast? + # long_description=orgparse.__doc__, keywords='org org-mode emacs', classifiers=[ diff --git a/orgparse/__init__.py b/src/orgparse/__init__.py similarity index 98% rename from orgparse/__init__.py rename to src/orgparse/__init__.py index e3f45cb..416a3b7 100644 --- a/orgparse/__init__.py +++ b/src/orgparse/__init__.py @@ -113,8 +113,6 @@ from .node import parse_lines, OrgEnv, OrgNode # todo basenode?? -__author__ = 'Takafumi Arakaki, Dmitrii Gerasimov' -__license__ = 'BSD License' __all__ = ["load", "loads", "loadi"] diff --git a/orgparse/date.py b/src/orgparse/date.py similarity index 100% rename from orgparse/date.py rename to src/orgparse/date.py diff --git a/orgparse/extra.py b/src/orgparse/extra.py similarity index 100% rename from orgparse/extra.py rename to src/orgparse/extra.py diff --git a/orgparse/inline.py b/src/orgparse/inline.py similarity index 100% rename from orgparse/inline.py rename to src/orgparse/inline.py diff --git a/orgparse/node.py b/src/orgparse/node.py similarity index 100% rename from orgparse/node.py rename to src/orgparse/node.py diff --git a/orgparse/py.typed b/src/orgparse/py.typed similarity index 100% rename from orgparse/py.typed rename to src/orgparse/py.typed diff --git a/orgparse/tests/__init__.py b/src/orgparse/tests/__init__.py similarity index 100% rename from orgparse/tests/__init__.py rename to src/orgparse/tests/__init__.py diff --git a/orgparse/tests/data/00_simple.org b/src/orgparse/tests/data/00_simple.org similarity index 100% rename from orgparse/tests/data/00_simple.org rename to src/orgparse/tests/data/00_simple.org diff --git a/orgparse/tests/data/00_simple.py b/src/orgparse/tests/data/00_simple.py similarity index 100% rename from orgparse/tests/data/00_simple.py rename to src/orgparse/tests/data/00_simple.py diff --git a/orgparse/tests/data/01_attributes.org b/src/orgparse/tests/data/01_attributes.org similarity index 100% rename from orgparse/tests/data/01_attributes.org rename to src/orgparse/tests/data/01_attributes.org diff --git a/orgparse/tests/data/01_attributes.py b/src/orgparse/tests/data/01_attributes.py similarity index 100% rename from orgparse/tests/data/01_attributes.py rename to src/orgparse/tests/data/01_attributes.py diff --git a/orgparse/tests/data/02_tree_struct.org b/src/orgparse/tests/data/02_tree_struct.org similarity index 100% rename from orgparse/tests/data/02_tree_struct.org rename to src/orgparse/tests/data/02_tree_struct.org diff --git a/orgparse/tests/data/02_tree_struct.py b/src/orgparse/tests/data/02_tree_struct.py similarity index 100% rename from orgparse/tests/data/02_tree_struct.py rename to src/orgparse/tests/data/02_tree_struct.py diff --git a/orgparse/tests/data/03_repeated_tasks.org b/src/orgparse/tests/data/03_repeated_tasks.org similarity index 100% rename from orgparse/tests/data/03_repeated_tasks.org rename to src/orgparse/tests/data/03_repeated_tasks.org diff --git a/orgparse/tests/data/03_repeated_tasks.py b/src/orgparse/tests/data/03_repeated_tasks.py similarity index 100% rename from orgparse/tests/data/03_repeated_tasks.py rename to src/orgparse/tests/data/03_repeated_tasks.py diff --git a/orgparse/tests/data/04_logbook.org b/src/orgparse/tests/data/04_logbook.org similarity index 100% rename from orgparse/tests/data/04_logbook.org rename to src/orgparse/tests/data/04_logbook.org diff --git a/orgparse/tests/data/04_logbook.py b/src/orgparse/tests/data/04_logbook.py similarity index 100% rename from orgparse/tests/data/04_logbook.py rename to src/orgparse/tests/data/04_logbook.py diff --git a/orgparse/tests/data/05_tags.org b/src/orgparse/tests/data/05_tags.org similarity index 100% rename from orgparse/tests/data/05_tags.org rename to src/orgparse/tests/data/05_tags.org diff --git a/orgparse/tests/data/05_tags.py b/src/orgparse/tests/data/05_tags.py similarity index 100% rename from orgparse/tests/data/05_tags.py rename to src/orgparse/tests/data/05_tags.py diff --git a/orgparse/tests/data/__init__.py b/src/orgparse/tests/data/__init__.py similarity index 100% rename from orgparse/tests/data/__init__.py rename to src/orgparse/tests/data/__init__.py diff --git a/orgparse/tests/test_data.py b/src/orgparse/tests/test_data.py similarity index 96% rename from orgparse/tests/test_data.py rename to src/orgparse/tests/test_data.py index 75e75e4..f315878 100644 --- a/orgparse/tests/test_data.py +++ b/src/orgparse/tests/test_data.py @@ -13,7 +13,8 @@ def load_data(path): """Load data from python file""" ns = {} # type: ignore - exec(Path(path).read_text(), ns) + # read_bytes() and compile hackery to avoid encoding issues (e.g. see 05_tags) + exec(compile(Path(path).read_bytes(), path, 'exec'), ns) return ns['data'] diff --git a/orgparse/tests/test_date.py b/src/orgparse/tests/test_date.py similarity index 100% rename from orgparse/tests/test_date.py rename to src/orgparse/tests/test_date.py diff --git a/orgparse/tests/test_hugedata.py b/src/orgparse/tests/test_hugedata.py similarity index 100% rename from orgparse/tests/test_hugedata.py rename to src/orgparse/tests/test_hugedata.py diff --git a/orgparse/tests/test_misc.py b/src/orgparse/tests/test_misc.py similarity index 100% rename from orgparse/tests/test_misc.py rename to src/orgparse/tests/test_misc.py diff --git a/orgparse/tests/test_rich.py b/src/orgparse/tests/test_rich.py similarity index 100% rename from orgparse/tests/test_rich.py rename to src/orgparse/tests/test_rich.py diff --git a/tox.ini b/tox.ini index 3adf49d..788cbb5 100644 --- a/tox.ini +++ b/tox.ini @@ -2,21 +2,39 @@ minversion = 3.7 # relies on the correct version of Python installed envlist = tests,mypy +# https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 +# hack to prevent .tox from crapping to the project directory +toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox [testenv] -passenv = CI,CI_* +# TODO how to get package name from setuptools? +package_name = "orgparse" +passenv = +# useful for tests to know they are running under ci + CI + CI_* +# respect user's cache dirs to prevent tox from crapping into project dir + MYPY_CACHE_DIR + PYTHONPYCACHEPREFIX + +# note: --use-pep517 here is necessary for tox --parallel flag to work properly +# otherwise it seems that it tries to modify .eggs dir in parallel and it fails [testenv:tests] commands = - pip install -e .[testing] - python -m pytest --ignore-glob='**/_py3compat.py' orgparse {posargs} + {envpython} -m pip install --use-pep517 -e .[testing] + # posargs allow test filtering, e.g. tox ... -- -k test_name + {envpython} -m pytest \ + --pyargs {[testenv]package_name} \ + {posargs} + [testenv:mypy] commands = - pip install -e .[linting] - python -m mypy --install-types --non-interactive \ - orgparse \ - # txt report is a bit more convenient to view on CI - --txt-report .coverage.mypy \ - --html-report .coverage.mypy \ - {posargs} + {envpython} -m pip install --use-pep517 -e .[linting] + {envpython} -m mypy --install-types --non-interactive \ + -p {[testenv]package_name} \ + # txt report is a bit more convenient to view on CI + --txt-report .coverage.mypy \ + --html-report .coverage.mypy \ + {posargs} From 5cb351abfd8445ad7834c4e7faeeecff9e584a85 Mon Sep 17 00:00:00 2001 From: karlicoss Date: Mon, 2 Oct 2023 23:29:38 +0100 Subject: [PATCH 81/90] general: switch to use pyproject.toml --- .ci/release | 2 +- .github/workflows/main.yml | 4 +-- pyproject.toml | 59 ++++++++++++++++++++++++++++++++++++++ setup.py | 55 ----------------------------------- 4 files changed, 62 insertions(+), 58 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.ci/release b/.ci/release index 95e41f4..6cff663 100755 --- a/.ci/release +++ b/.ci/release @@ -42,7 +42,7 @@ def main() -> None: if dist.exists(): shutil.rmtree(dist) - check_call('python3 setup.py sdist bdist_wheel', shell=True) + check_call(['python3', '-m', 'build']) TP = 'TWINE_PASSWORD' password = os.environ.get(TP) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 89338bd..271624d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -80,7 +80,7 @@ jobs: if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master' env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }} - run: pip3 install --user wheel twine && .ci/release --test + run: pip3 install --user --upgrade build twine && .ci/release --test - name: 'release to pypi' # always deploy tags to release pypi @@ -88,4 +88,4 @@ jobs: if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags') env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - run: pip3 install --user wheel twine && .ci/release + run: pip3 install --user --upgrade build twine && .ci/release diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7fae398 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,59 @@ +[project] +dynamic = ["version"] # version is managed by setuptools_scm +name = "orgparse" +description = "orgparse - Emacs org-mode parser in Python" +license = {file = "LICENSE"} +authors = [ + {name = "Takafumi Arakaki (@tkf)", email = "aka.tkf@gmail.com"}, + {name = "Dmitrii Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"}, +] +maintainers = [ + {name = "Dmitrii Gerasimov (@karlicoss)", email = "karlicoss@gmail.com"}, +] +keywords = ["org", "org-mode", "emacs"] +# see: http://pypi.python.org/pypi?%3Aaction=list_classifiers +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: BSD License", + "Topic :: Text Processing :: Markup", +] +# TODO add it back later, perhaps via ast? +# long_description=orgparse.__doc__, + +[project.urls] +Homepage = "https://github.com/karlicoss/orgparse" + +[project.optional-dependencies] +testing = [ + "pytest", +] +linting = [ + "pytest", + "mypy", + "lxml", # for mypy html coverage +] + + +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +version_scheme = "python-simplified-semver" +local_scheme = "dirty-tag" + + +# nice things about pyproject.toml +# - zip_safe=False isn't neccessary anymore +# - correctly discovers namespace packages by defuilt? +# - correctly handles py.typed by default? +# - handles src layout automatically https://setuptools.pypa.io/en/latest/userguide/package_discovery.html#src-layout + +# things I'm not sure about yet +# - avoiding dupliation/variable reuse? +# - file/git dependencies? +# - unclear how to specify namespace package order https://github.com/seanbreckenridge/reorder_editable/issues/2 + +# TODO +# - maybe it has a nicer pypi upload system? not sure +# e.g. possibly use hatch/flit/pdb/poetry -- but not sure what's the benefit tbh diff --git a/setup.py b/setup.py deleted file mode 100644 index 5d500b1..0000000 --- a/setup.py +++ /dev/null @@ -1,55 +0,0 @@ -# see https://github.com/karlicoss/pymplate for up-to-date reference -# -from setuptools import setup, find_namespace_packages # type: ignore - - -def main(): - pkgs = find_namespace_packages('src') - pkg = min(pkgs) - - setup( - name=pkg, - use_scm_version={ - 'version_scheme': 'python-simplified-semver', - 'local_scheme': 'dirty-tag', - }, - setup_requires=['setuptools_scm'], - - zip_safe=False, - - packages=pkgs, - package_dir={'': 'src'}, - package_data={ - pkg: ['py.typed'], # todo need the rest as well?? - 'orgparse.tests.data': ['*.org'], - }, - - author='Takafumi Arakaki, Dmitrii Gerasimov', - author_email='aka.tkf@gmail.com', - maintainer='Dima Gerasimov (@karlicoss)', - maintainer_email='karlicoss@gmail.com', - - url='https://github.com/karlicoss/orgparse', - license='BSD License', - - description='orgparse - Emacs org-mode parser in Python', - # TODO add it back later, perhaps via ast? - # long_description=orgparse.__doc__, - - keywords='org org-mode emacs', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'License :: OSI Approved :: BSD License', - 'Topic :: Text Processing :: Markup', - # see: http://pypi.python.org/pypi?%3Aaction=list_classifiers - ], - - extras_require={ - 'testing': ['pytest'], - 'linting': ['pytest', 'mypy', 'lxml'], # lxml for mypy coverage report - }, - ) - - -if __name__ == '__main__': - main() From ab6d3fdeadcde9e62b8d2add0e1a25541f4637ac Mon Sep 17 00:00:00 2001 From: karlicoss Date: Wed, 4 Oct 2023 18:45:26 +0100 Subject: [PATCH 82/90] ci: use python 3.12 instead of rc --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 271624d..fdb7e7d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,7 +25,7 @@ jobs: fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12.0-rc.3'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] # vvv just an example of excluding stuff from matrix # exclude: [{platform: macos-latest, python-version: '3.6'}] From fe6f4a973db7d0bda968dc4076212e30a4ea7ea6 Mon Sep 17 00:00:00 2001 From: karlicoss Date: Wed, 4 Oct 2023 19:00:57 +0100 Subject: [PATCH 83/90] remove hack for python3.11 on osx --- .github/workflows/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fdb7e7d..80b66b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,9 +34,6 @@ jobs: steps: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }} - # hmm somehow only seems necessary for 3.11 on osx?? - run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH - uses: actions/setup-python@v4 with: From da56aae64a6373ae8bab2dde9dc756f904f1d8f8 Mon Sep 17 00:00:00 2001 From: karlicoss Date: Wed, 4 Oct 2023 20:36:00 +0100 Subject: [PATCH 84/90] ci: sync configuration with pymplate --- pyproject.toml | 1 + ruff.toml | 25 +++++++++++++++++++++++++ src/orgparse/node.py | 2 +- tox.ini | 12 +++++++++--- 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 ruff.toml diff --git a/pyproject.toml b/pyproject.toml index 7fae398..30719be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ testing = [ ] linting = [ "pytest", + "ruff", "mypy", "lxml", # for mypy html coverage ] diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..0be93e0 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,25 @@ +ignore = [ +### too opinionated style checks + "E501", # too long lines + "E702", # Multiple statements on one line (semicolon) + "E731", # assigning lambda instead of using def + "E741", # Ambiguous variable name: `l` + "E742", # Ambiguous class name: `O + "E401", # Multiple imports on one line + "F403", # import *` used; unable to detect undefined names +### + +### + "E722", # Do not use bare `except` ## Sometimes it's useful for defensive imports and that sort of thing.. + "F811", # Redefinition of unused # this gets in the way of pytest fixtures (e.g. in cachew) + +## might be nice .. but later and I don't wanna make it strict + "E402", # Module level import not at top of file + +### maybe consider these soon +# sometimes it's useful to give a variable a name even if we don't use it as a documentation +# on the other hand, often is a sign of error + "F841", # Local variable `count` is assigned to but never used + "F401", # imported but unused +### +] diff --git a/src/orgparse/node.py b/src/orgparse/node.py index 154a3e8..7ed1cdb 100644 --- a/src/orgparse/node.py +++ b/src/orgparse/node.py @@ -832,7 +832,7 @@ def level(self): :rtype: int """ - raise NotImplemented + raise NotImplementedError def _get_tags(self, inher=False) -> Set[str]: """ diff --git a/tox.ini b/tox.ini index 788cbb5..dc04b0e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,10 @@ [tox] -minversion = 3.7 +minversion = 3.21 # relies on the correct version of Python installed -envlist = tests,mypy +envlist = ruff,tests,mypy # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 # hack to prevent .tox from crapping to the project directory -toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox +toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox [testenv] # TODO how to get package name from setuptools? @@ -18,6 +18,12 @@ passenv = PYTHONPYCACHEPREFIX +[testenv:ruff] +commands = + {envpython} -m pip install --use-pep517 -e .[linting] + {envpython} -m ruff src/ + + # note: --use-pep517 here is necessary for tox --parallel flag to work properly # otherwise it seems that it tries to modify .eggs dir in parallel and it fails [testenv:tests] From db837988100fb8c67f4689d8cb1ff35e87768b48 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 20 May 2025 20:46:56 +0100 Subject: [PATCH 85/90] general: update CI files - set min version to python 3.9 - switch build backend to hatch - publish via uv --- .ci/release-uv | 60 +++++++++++++++ .ci/run | 16 ++-- .github/workflows/main.yml | 46 +++++++----- conftest.py | 11 ++- mypy.ini | 11 ++- pyproject.toml | 39 +++++----- pytest.ini | 7 ++ ruff.toml | 127 ++++++++++++++++++++++++++++++-- src/orgparse/extra.py | 4 +- src/orgparse/tests/test_data.py | 2 +- tox.ini | 29 +++++--- 11 files changed, 276 insertions(+), 76 deletions(-) create mode 100755 .ci/release-uv diff --git a/.ci/release-uv b/.ci/release-uv new file mode 100755 index 0000000..c56697c --- /dev/null +++ b/.ci/release-uv @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +''' +Deploys Python package onto [[https://pypi.org][PyPi]] or [[https://test.pypi.org][test PyPi]]. + +- running manually + + You'll need =UV_PUBLISH_TOKEN= env variable + +- running on Github Actions + + Instead of env variable, relies on configuring github as Trusted publisher (https://docs.pypi.org/trusted-publishers/) -- both for test and regular pypi + + It's running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]]. + Packages are deployed on: + - every master commit, onto test pypi + - every new tag, onto production pypi +''' + +UV_PUBLISH_TOKEN = 'UV_PUBLISH_TOKEN' + +import argparse +import os +import shutil +from pathlib import Path +from subprocess import check_call + +is_ci = os.environ.get('CI') is not None + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument('--use-test-pypi', action='store_true') + args = p.parse_args() + + publish_url = ['--publish-url', 'https://test.pypi.org/legacy/'] if args.use_test_pypi else [] + + root = Path(__file__).absolute().parent.parent + os.chdir(root) # just in case + + if is_ci: + # see https://github.com/actions/checkout/issues/217 + check_call('git fetch --prune --unshallow'.split()) + + # TODO ok, for now uv won't remove dist dir if it already exists + # https://github.com/astral-sh/uv/issues/10293 + dist = root / 'dist' + if dist.exists(): + shutil.rmtree(dist) + + # todo what is --force-pep517? + check_call(['uv', 'build']) + + if not is_ci: + # CI relies on trusted publishers so doesn't need env variable + assert UV_PUBLISH_TOKEN in os.environ, f'no {UV_PUBLISH_TOKEN} passed' + + check_call(['uv', 'publish', *publish_url]) + + +if __name__ == '__main__': + main() diff --git a/.ci/run b/.ci/run index b2c184d..c881818 100755 --- a/.ci/run +++ b/.ci/run @@ -11,6 +11,8 @@ if ! command -v sudo; then } fi +# --parallel-live to show outputs while it's running +tox_cmd='run-parallel --parallel-live' if [ -n "${CI-}" ]; then # install OS specific stuff here case "$OSTYPE" in @@ -20,7 +22,8 @@ if [ -n "${CI-}" ]; then ;; cygwin* | msys* | win*) # windows - : + # ugh. parallel stuff seems super flaky under windows, some random failures, "file used by other process" and crap like that + tox_cmd='run' ;; *) # must be linux? @@ -29,12 +32,5 @@ if [ -n "${CI-}" ]; then esac fi - -PY_BIN="python3" -# some systems might have python pointing to python3 -if ! command -v python3 &> /dev/null; then - PY_BIN="python" -fi - -"$PY_BIN" -m pip install --user tox -"$PY_BIN" -m tox --parallel --parallel-live "$@" +# NOTE: expects uv installed +uv tool run --with tox-uv tox $tox_cmd "$@" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 80b66b2..3599e6a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,6 @@ on: branches: '*' tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug: - # P.S. fuck made up yaml DSLs. pull_request: # needed to trigger on others' PRs # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them". workflow_dispatch: # needed to trigger workflows manually @@ -25,23 +24,31 @@ jobs: fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] # vvv just an example of excluding stuff from matrix # exclude: [{platform: macos-latest, python-version: '3.6'}] runs-on: ${{ matrix.platform }} + # useful for 'optional' pipelines + # continue-on-error: ${{ matrix.platform == 'windows-latest' }} + steps: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 with: - python-version: ${{ matrix.python-version }} + submodules: recursive + fetch-depth: 0 # nicer to have all git history when debugging/for tests - - uses: actions/checkout@v3 + - uses: actions/setup-python@v5 with: - submodules: recursive + python-version: ${{ matrix.python-version }} + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: false # we don't have lock files, so can't use them as cache key - uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} @@ -50,8 +57,9 @@ jobs: - run: bash .ci/run - if: matrix.platform == 'ubuntu-latest' # no need to compute coverage for other platforms - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: + include-hidden-files: true name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} path: .coverage.mypy/ @@ -59,30 +67,32 @@ jobs: pypi: runs-on: ubuntu-latest needs: [build] # add all other jobs here - + permissions: + # necessary for Trusted Publishing + id-token: write steps: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 with: - python-version: '3.8' + submodules: recursive - - uses: actions/checkout@v3 + - uses: actions/setup-python@v5 with: - submodules: recursive + python-version: '3.10' + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: false # we don't have lock files, so can't use them as cache key - name: 'release to test pypi' # always deploy merged master to test pypi if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master' - env: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }} - run: pip3 install --user --upgrade build twine && .ci/release --test + run: .ci/release-uv --use-test-pypi - name: 'release to pypi' # always deploy tags to release pypi # NOTE: release tags are guarded by on: push: tags on the top if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags') - env: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - run: pip3 install --user --upgrade build twine && .ci/release + run: .ci/release-uv diff --git a/conftest.py b/conftest.py index 466da24..91a43a3 100644 --- a/conftest.py +++ b/conftest.py @@ -2,6 +2,7 @@ # without it, pytest can't discover the package root for some reason # also see https://github.com/karlicoss/pytest_namespace_pkgs for more +import os import pathlib from typing import Optional @@ -24,6 +25,10 @@ def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]: for parent in result.parents: if str(parent) in namespace_pkg_dirs: return parent + if os.name == 'nt': + # ??? for some reason on windows it is trying to call this against conftest? but not on linux/osx + if path.name == 'conftest.py': + return resolve_pkg_path_orig(path) raise RuntimeError("Couldn't determine path for ", path) _pytest.pathlib.resolve_package_path = resolve_package_path @@ -34,5 +39,9 @@ def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]: # not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure.. search_pypath_orig = _pytest.main.search_pypath def search_pypath(module_name: str) -> str: - return str(root_dir) + mpath = root_dir / module_name.replace('.', os.sep) + if not mpath.is_dir(): + mpath = mpath.with_suffix('.py') + assert mpath.exists(), mpath # just in case + return str(mpath) _pytest.main.search_pypath = search_pypath diff --git a/mypy.ini b/mypy.ini index 5a21a85..7b1e535 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,14 +1,17 @@ [mypy] -namespace_packages = True pretty = True show_error_context = True -show_error_codes = True show_column_numbers = True show_error_end = True -warn_unused_ignores = True + check_untyped_defs = True -enable_error_code = possibly-undefined + +# see https://mypy.readthedocs.io/en/stable/error_code_list2.html +warn_redundant_casts = True strict_equality = True +warn_unused_ignores = True +enable_error_code = deprecated,redundant-expr,possibly-undefined,truthy-bool,truthy-iterable,ignore-without-code,unused-awaitable + # an example of suppressing # [mypy-my.config.repos.pdfannots.pdfannots] diff --git a/pyproject.toml b/pyproject.toml index 30719be..359a928 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,9 @@ [project] -dynamic = ["version"] # version is managed by setuptools_scm +dynamic = ["version"] # version is managed by build backend name = "orgparse" +dependencies = [ +] +requires-python = ">=3.9" description = "orgparse - Emacs org-mode parser in Python" license = {file = "LICENSE"} authors = [ @@ -24,10 +27,8 @@ classifiers = [ Homepage = "https://github.com/karlicoss/orgparse" [project.optional-dependencies] +[dependency-groups] testing = [ - "pytest", -] -linting = [ "pytest", "ruff", "mypy", @@ -35,26 +36,20 @@ linting = [ ] -[build-system] -requires = ["setuptools", "setuptools-scm"] -build-backend = "setuptools.build_meta" -[tool.setuptools_scm] -version_scheme = "python-simplified-semver" -local_scheme = "dirty-tag" +# workaround for error during uv publishing +# see https://github.com/astral-sh/uv/issues/9513#issuecomment-2519527822 +[tool.setuptools] +license-files = [] -# nice things about pyproject.toml -# - zip_safe=False isn't neccessary anymore -# - correctly discovers namespace packages by defuilt? -# - correctly handles py.typed by default? -# - handles src layout automatically https://setuptools.pypa.io/en/latest/userguide/package_discovery.html#src-layout +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" -# things I'm not sure about yet -# - avoiding dupliation/variable reuse? -# - file/git dependencies? -# - unclear how to specify namespace package order https://github.com/seanbreckenridge/reorder_editable/issues/2 +# unfortunately have to duplicate project name here atm, see https://github.com/pypa/hatch/issues/1894 +[tool.hatch.build.targets.wheel] +packages = ["src/orgparse"] -# TODO -# - maybe it has a nicer pypi upload system? not sure -# e.g. possibly use hatch/flit/pdb/poetry -- but not sure what's the benefit tbh +[tool.hatch.version] +source = "vcs" diff --git a/pytest.ini b/pytest.ini index 20c3704..226488b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,7 +1,14 @@ [pytest] # discover files that don't follow test_ naming. Useful to keep tests along with the source code python_files = *.py + +# this setting only impacts package/module naming under pytest, not the discovery +consider_namespace_packages = true + addopts = + # prevent pytest cache from being created... it craps into project dir and I never use it anyway + -p no:cacheprovider + # -rap to print tests summary even when they are successful -rap --verbose diff --git a/ruff.toml b/ruff.toml index 0be93e0..f6fd8b7 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,4 +1,54 @@ -ignore = [ +lint.extend-select = [ + "F", # flakes rules -- default, but extend just in case + "E", # pycodestyle -- default, but extend just in case + "W", # various warnings + + "B", # 'bugbear' set -- various possible bugs + "C4", # flake8-comprehensions -- unnecessary list/map/dict calls + "COM", # trailing commas + "EXE", # various checks wrt executable files + "I", # sort imports + "ICN", # various import conventions + "FBT", # detect use of boolean arguments + "FURB", # various rules + "PERF", # various potential performance speedups + "PD", # pandas rules + "PIE", # 'misc' lints + "PLC", # pylint convention rules + "PLR", # pylint refactor rules + "PLW", # pylint warnings + "PT", # pytest stuff + "PYI", # various type hinting rules + "RET", # early returns + "RUF", # various ruff-specific rules + "TID", # various imports suggestions + "TRY", # various exception handling rules + "UP", # detect deprecated python stdlib stuff + "FA", # suggest using from __future__ import annotations + "PTH", # pathlib migration + "ARG", # unused argument checks + "A", # builtin shadowing + "G", # logging stuff + # "EM", # TODO hmm could be helpful to prevent duplicate err msg in traceback.. but kinda annoying + + # "ALL", # uncomment this to check for new rules! +] + +# Preserve types, even if a file imports `from __future__ import annotations` +# we need this for cachew to work with HPI types on 3.9 +# can probably remove after 3.10? +lint.pyupgrade.keep-runtime-typing = true + +lint.ignore = [ + "D", # annoying nags about docstrings + "N", # pep naming + "TCH", # type checking rules, mostly just suggests moving imports under TYPE_CHECKING + "S", # bandit (security checks) -- tends to be not very useful, lots of nitpicks + "DTZ", # datetimes checks -- complaining about missing tz and mostly false positives + "FIX", # complains about fixmes/todos -- annoying + "TD", # complains about todo formatting -- too annoying + "ANN", # missing type annotations? seems way to strict though + ### too opinionated style checks "E501", # too long lines "E702", # Multiple statements on one line (semicolon) @@ -16,10 +66,75 @@ ignore = [ ## might be nice .. but later and I don't wanna make it strict "E402", # Module level import not at top of file -### maybe consider these soon -# sometimes it's useful to give a variable a name even if we don't use it as a documentation -# on the other hand, often is a sign of error - "F841", # Local variable `count` is assigned to but never used - "F401", # imported but unused + "RUF100", # unused noqa -- handle later + "RUF012", # mutable class attrs should be annotated with ClassVar... ugh pretty annoying for user configs + +### these are just nitpicky, we usually know better + "PLR0911", # too many return statements + "PLR0912", # too many branches + "PLR0913", # too many function arguments + "PLR0915", # too many statements + "PLR1714", # consider merging multiple comparisons + "PLR2044", # line with empty comment + "PLR5501", # use elif instead of else if + "PLR2004", # magic value in comparison -- super annoying in tests ### + "PLR0402", # import X.Y as Y -- TODO maybe consider enabling it, but double check + + "B009", # calling gettattr with constant attribute -- this is useful to convince mypy + "B010", # same as above, but setattr + "B011", # complains about assert False + "B017", # pytest.raises(Exception) + "B023", # seems to result in false positives? + "B028", # suggest using explicit stacklevel? TODO double check later, but not sure it's useful + + # complains about useless pass, but has sort of a false positive if the function has a docstring? + # this is common for click entrypoints (e.g. in __main__), so disable + "PIE790", + + # a bit too annoying, offers to convert for loops to list comprehension + # , which may heart readability + "PERF401", + + # suggests no using exception in for loops + # we do use this technique a lot, plus in 3.11 happy path exception handling is "zero-cost" + "PERF203", + + "RET504", # unnecessary assignment before returning -- that can be useful for readability + "RET505", # unnecessary else after return -- can hurt readability + + "PLW0603", # global variable update.. we usually know why we are doing this + "PLW2901", # for loop variable overwritten, usually this is intentional + + "PT011", # pytest raises should is too broad + "PT012", # pytest raises should contain a single statement + + "COM812", # trailing comma missing -- mostly just being annoying with long multiline strings + + "PD901", # generic variable name df + + "TRY003", # suggests defining exception messages in exception class -- kinda annoying + "TRY004", # prefer TypeError -- don't see the point + "TRY201", # raise without specifying exception name -- sometimes hurts readability + "TRY400", # TODO double check this, might be useful + "TRY401", # redundant exception in logging.exception call? TODO double check, might result in excessive logging + + "PGH", # TODO force error code in mypy instead? although it also has blanket noqa rule + + "TID252", # Prefer absolute imports over relative imports from parent modules + + "UP038", # suggests using | (union) in isisntance checks.. but it results in slower code + + ## too annoying + "T20", # just complains about prints and pprints + "Q", # flake quotes, too annoying + "C90", # some complexity checking + "G004", # logging statement uses f string + "ERA001", # commented out code + "SLF001", # private member accessed + "BLE001", # do not catch 'blind' Exception + "INP001", # complains about implicit namespace packages + "SIM", # some if statements crap + "RSE102", # complains about missing parens in exceptions + ## ] diff --git a/src/orgparse/extra.py b/src/orgparse/extra.py index cd51aba..5fefcd6 100644 --- a/src/orgparse/extra.py +++ b/src/orgparse/extra.py @@ -96,7 +96,7 @@ def emit() -> Rich: if last is Gap: res = Gap() elif last is Table: - res = Table(group) # type: ignore + res = Table(group) # type: ignore[assignment] else: raise RuntimeError(f'Unexpected type {last}') group = [] @@ -106,7 +106,7 @@ def emit() -> Rich: if RE_TABLE_ROW.match(line) or RE_TABLE_SEPARATOR.match(line): cur = Table else: - cur = Gap # type: ignore + cur = Gap # type: ignore[assignment] if cur is not last: if len(group) > 0: yield emit() diff --git a/src/orgparse/tests/test_data.py b/src/orgparse/tests/test_data.py index f315878..60e4db0 100644 --- a/src/orgparse/tests/test_data.py +++ b/src/orgparse/tests/test_data.py @@ -12,7 +12,7 @@ def load_data(path): """Load data from python file""" - ns = {} # type: ignore + ns = {} # type: ignore[var-annotated] # read_bytes() and compile hackery to avoid encoding issues (e.g. see 05_tags) exec(compile(Path(path).read_bytes(), path, 'exec'), ns) return ns['data'] diff --git a/tox.ini b/tox.ini index dc04b0e..681618b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,8 @@ [tox] minversion = 3.21 # relies on the correct version of Python installed -envlist = ruff,tests,mypy +envlist = tests,mypy +# FIXME will fix ruff in a later commit # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 # hack to prevent .tox from crapping to the project directory toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox @@ -11,24 +12,28 @@ toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox package_name = "orgparse" passenv = # useful for tests to know they are running under ci - CI - CI_* + CI + CI_* # respect user's cache dirs to prevent tox from crapping into project dir - MYPY_CACHE_DIR - PYTHONPYCACHEPREFIX + PYTHONPYCACHEPREFIX + MYPY_CACHE_DIR + RUFF_CACHE_DIR + +# default is 'editable', in which tox builds wheel first for some reason? not sure if makes much sense +package = uv-editable [testenv:ruff] +skip_install = true +dependency_groups = testing commands = - {envpython} -m pip install --use-pep517 -e .[linting] - {envpython} -m ruff src/ + {envpython} -m ruff check src/ \ + {posargs} -# note: --use-pep517 here is necessary for tox --parallel flag to work properly -# otherwise it seems that it tries to modify .eggs dir in parallel and it fails [testenv:tests] +dependency_groups = testing commands = - {envpython} -m pip install --use-pep517 -e .[testing] # posargs allow test filtering, e.g. tox ... -- -k test_name {envpython} -m pytest \ --pyargs {[testenv]package_name} \ @@ -36,9 +41,9 @@ commands = [testenv:mypy] +dependency_groups = testing commands = - {envpython} -m pip install --use-pep517 -e .[linting] - {envpython} -m mypy --install-types --non-interactive \ + {envpython} -m mypy --no-install-types \ -p {[testenv]package_name} \ # txt report is a bit more convenient to view on CI --txt-report .coverage.mypy \ From 914b492ea58cbef1f7b4d32d3e7506de193a551f Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 20 May 2025 21:56:12 +0100 Subject: [PATCH 86/90] general: fix all ruff stuff + more mypy coverage --- ruff.toml | 2 +- src/orgparse/__init__.py | 9 +- src/orgparse/date.py | 116 +++++++----- src/orgparse/extra.py | 20 +- src/orgparse/node.py | 189 ++++++++++--------- src/orgparse/tests/data/00_simple.py | 24 ++- src/orgparse/tests/data/01_attributes.py | 105 ++++++----- src/orgparse/tests/data/02_tree_struct.py | 14 +- src/orgparse/tests/data/03_repeated_tasks.py | 15 +- src/orgparse/tests/data/04_logbook.py | 8 +- src/orgparse/tests/data/05_tags.py | 13 +- src/orgparse/tests/test_data.py | 16 +- src/orgparse/tests/test_date.py | 11 +- src/orgparse/tests/test_hugedata.py | 7 +- src/orgparse/tests/test_misc.py | 25 +-- src/orgparse/tests/test_rich.py | 6 +- tox.ini | 3 +- 17 files changed, 310 insertions(+), 273 deletions(-) diff --git a/ruff.toml b/ruff.toml index f6fd8b7..0fa381a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -25,7 +25,7 @@ lint.extend-select = [ "TRY", # various exception handling rules "UP", # detect deprecated python stdlib stuff "FA", # suggest using from __future__ import annotations - "PTH", # pathlib migration + # "PTH", # pathlib migration # FIXME do later.. a bit overwhelming "ARG", # unused argument checks "A", # builtin shadowing "G", # logging stuff diff --git a/src/orgparse/__init__.py b/src/orgparse/__init__.py index 416a3b7..d699f4f 100644 --- a/src/orgparse/__init__.py +++ b/src/orgparse/__init__.py @@ -106,14 +106,13 @@ """ # [[[end]]] -from io import IOBase +from collections.abc import Iterable from pathlib import Path -from typing import Iterable, Union, Optional, TextIO +from typing import Optional, TextIO, Union +from .node import OrgEnv, OrgNode, parse_lines # todo basenode?? -from .node import parse_lines, OrgEnv, OrgNode # todo basenode?? - -__all__ = ["load", "loads", "loadi"] +__all__ = ["load", "loadi", "loads"] def load(path: Union[str, Path, TextIO], env: Optional[OrgEnv] = None) -> OrgNode: diff --git a/src/orgparse/date.py b/src/orgparse/date.py index dd407b7..ccaa5c3 100644 --- a/src/orgparse/date.py +++ b/src/orgparse/date.py @@ -1,22 +1,25 @@ +from __future__ import annotations + import datetime import re -from typing import Union, Tuple, Optional, List +from datetime import timedelta +from typing import Optional, Union DateIsh = Union[datetime.date, datetime.datetime] -def total_seconds(td): +def total_seconds(td: timedelta) -> float: """Equivalent to `datetime.timedelta.total_seconds`.""" return float(td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10 ** 6 -def total_minutes(td): +def total_minutes(td: timedelta) -> float: """Alias for ``total_seconds(td) / 60``.""" return total_seconds(td) / 60 -def gene_timestamp_regex(brtype, prefix=None, nocookie=False): +def gene_timestamp_regex(brtype: str, prefix: str | None = None, *, nocookie: bool = False) -> str: """ Generate timestamp regex for active/inactive/nobrace brace type @@ -84,15 +87,15 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): elif brtype == 'nobrace': (bo, bc) = ('', '') else: - raise ValueError("brtype='{0!r}' is invalid".format(brtype)) + raise ValueError(f"brtype='{brtype!r}' is invalid") if brtype == 'nobrace': ignore = r'[\s\w]' else: - ignore = '[^{bc}]'.format(bc=bc) + ignore = f'[^{bc}]' if prefix is None: - prefix = '{0}_'.format(brtype) + prefix = f'{brtype}_' regex_date_time = r""" (?P<{prefix}year>\d{{4}}) - @@ -133,7 +136,7 @@ def gene_timestamp_regex(brtype, prefix=None, nocookie=False): return regex.format(prefix=prefix, ignore=ignore) -def date_time_format(date) -> str: +def date_time_format(date: DateIsh) -> str: """ Format a date or datetime in default org format @@ -165,7 +168,10 @@ def is_same_day(date0, date1) -> bool: re.VERBOSE) -class OrgDate(object): +_Repeater = tuple[str, int, str] + + +class OrgDate: _active_default = True """ @@ -184,8 +190,14 @@ class OrgDate(object): """ _allow_short_range = True - def __init__(self, start, end=None, active=None, repeater=None, - warning=None): + def __init__( + self, + start, + end=None, + active: bool | None = None, + repeater: _Repeater | None = None, + warning: _Repeater | None = None, + ) -> None: """ Create :class:`OrgDate` object @@ -242,21 +254,23 @@ def _to_date(date) -> DateIsh: raise ValueError( "Automatic conversion to the datetime object " "requires at least 3 elements in the tuple. " - "Only {0} elements are in the given tuple '{1}'." - .format(len(date), date)) + f"Only {len(date)} elements are in the given tuple '{date}'." + ) elif isinstance(date, (int, float)): return datetime.datetime.fromtimestamp(date) else: return date @staticmethod - def _date_to_tuple(date): + def _date_to_tuple(date: DateIsh) -> tuple[int, ...]: if isinstance(date, datetime.datetime): return tuple(date.timetuple()[:6]) elif isinstance(date, datetime.date): return tuple(date.timetuple()[:3]) + else: + raise RuntimeError(f"can't happen: {date}") - def __repr__(self): + def __repr__(self) -> str: args = [ self.__class__.__name__, self._date_to_tuple(self.start), @@ -269,9 +283,9 @@ def __repr__(self): args.pop() if len(args) > 3 and args[3] is None: args[3] = self._active_default - return '{0}({1})'.format(args[0], ', '.join(map(repr, args[1:]))) + return '{}({})'.format(args[0], ', '.join(map(repr, args[1:]))) - def __str__(self): + def __str__(self) -> str: fence = ("<", ">") if self.is_active() else ("[", "]") start = date_time_format(self.start) @@ -279,26 +293,26 @@ def __str__(self): if self.has_end(): if self._allow_short_range and is_same_day(self.start, self.end): - start += "--%s" % self.end.strftime("%H:%M") + start += "--{}".format(self.end.strftime("%H:%M")) else: end = date_time_format(self.end) - if self._repeater: - start += " %s%d%s" % self._repeater - if self._warning: - start += " %s%d%s" % self._warning - ret = "%s%s%s" % (fence[0], start, fence[1]) + if self._repeater is not None: + (x, y, z) = self._repeater + start += f" {x}{y}{z}" + if self._warning is not None: + (x, y, z) = self._warning + start += f" {x}{y}{z}" + ret = f"{fence[0]}{start}{fence[1]}" if end: - ret += "--%s%s%s" % (fence[0], end, fence[1]) + ret += f"--{fence[0]}{end}{fence[1]}" return ret - def __nonzero__(self): + def __bool__(self) -> bool: return bool(self._start) - __bool__ = __nonzero__ # PY3 - - def __eq__(self, other): + def __eq__(self, other) -> bool: if (isinstance(other, OrgDate) and self._start is None and other._start is None): @@ -309,7 +323,7 @@ def __eq__(self, other): self._active == other._active) @property - def start(self): + def start(self) -> DateIsh: """ Get date or datetime object @@ -322,7 +336,7 @@ def start(self): return self._start @property - def end(self): + def end(self) -> DateIsh: """ Get date or datetime object @@ -404,11 +418,11 @@ def _as_datetime(date) -> datetime.datetime: return date @staticmethod - def _daterange_from_groupdict(dct, prefix='') -> Tuple[List, Optional[List]]: + def _daterange_from_groupdict(dct, prefix='') -> tuple[list, Optional[list]]: start_keys = ['year', 'month', 'day', 'hour' , 'min'] end_keys = ['year', 'month', 'day', 'end_hour', 'end_min'] start_range = list(map(int, filter(None, (dct[prefix + k] for k in start_keys)))) - end_range: Optional[List] + end_range: Optional[list] end_range = list(map(int, filter(None, (dct[prefix + k] for k in end_keys)))) if len(end_range) < len(end_keys): end_range = None @@ -419,7 +433,7 @@ def _datetuple_from_groupdict(cls, dct, prefix=''): return cls._daterange_from_groupdict(dct, prefix=prefix)[0] @classmethod - def list_from_str(cls, string: str) -> List['OrgDate']: + def list_from_str(cls, string: str) -> list[OrgDate]: """ Parse string and return a list of :class:`OrgDate` objects @@ -447,8 +461,8 @@ def list_from_str(cls, string: str) -> List['OrgDate']: prefix = 'inactive_' active = False rangedash = '--[' - repeater: Optional[Tuple[str, int, str]] = None - warning: Optional[Tuple[str, int, str]] = None + repeater: Optional[tuple[str, int, str]] = None + warning: Optional[tuple[str, int, str]] = None if mdict[prefix + 'repeatpre'] is not None: keys = [prefix + 'repeat' + suffix for suffix in cookie_suffix] values = [mdict[k] for k in keys] @@ -471,12 +485,12 @@ def list_from_str(cls, string: str) -> List['OrgDate']: odate = cls( *cls._daterange_from_groupdict(mdict, prefix), active=active, repeater=repeater, warning=warning) - return [odate] + cls.list_from_str(rest) + return [odate, *cls.list_from_str(rest)] else: return [] @classmethod - def from_str(cls, string): + def from_str(cls, string: str) -> OrgDate: """ Parse string and return an :class:`OrgDate` objects. @@ -500,7 +514,7 @@ def from_str(cls, string): def compile_sdc_re(sdctype): brtype = 'inactive' if sdctype == 'CLOSED' else 'active' return re.compile( - r'^(?!\#).*{0}:\s+{1}'.format( + r'^(?!\#).*{}:\s+{}'.format( sdctype, gene_timestamp_regex(brtype, prefix='', nocookie=True)), re.VERBOSE) @@ -528,8 +542,8 @@ def from_str(cls, string): end_dict.update({'hour': end_hour, 'min': end_min}) end = cls._datetuple_from_groupdict(end_dict) cookie_suffix = ['pre', 'num', 'dwmy'] - repeater: Optional[Tuple[str, int, str]] = None - warning: Optional[Tuple[str, int, str]] = None + repeater: Optional[tuple[str, int, str]] = None + warning: Optional[tuple[str, int, str]] = None prefix = '' if mdict[prefix + 'repeatpre'] is not None: keys = [prefix + 'repeat' + suffix for suffix in cookie_suffix] @@ -588,7 +602,7 @@ def __init__(self, start, end=None, duration=None, active=None): """ Create OrgDateClock object """ - super(OrgDateClock, self).__init__(start, end, active=active) + super().__init__(start, end, active=active) self._duration = duration @property @@ -625,7 +639,7 @@ def is_duration_consistent(self): self._duration == total_minutes(self.duration)) @classmethod - def from_str(cls, line: str) -> 'OrgDateClock': + def from_str(cls, line: str) -> OrgDateClock: """ Get CLOCK from given string. @@ -674,26 +688,26 @@ class OrgDateRepeatedTask(OrgDate): _active_default = False - def __init__(self, start, before, after, active=None): - super(OrgDateRepeatedTask, self).__init__(start, active=active) + def __init__(self, start, before: str, after: str, active=None) -> None: + super().__init__(start, active=active) self._before = before self._after = after - def __repr__(self): - args = [self._date_to_tuple(self.start), self.before, self.after] + def __repr__(self) -> str: + args: list = [self._date_to_tuple(self.start), self.before, self.after] if self._active is not self._active_default: args.append(self._active) - return '{0}({1})'.format( + return '{}({})'.format( self.__class__.__name__, ', '.join(map(repr, args))) - def __eq__(self, other): - return super(OrgDateRepeatedTask, self).__eq__(other) and \ + def __eq__(self, other) -> bool: + return super().__eq__(other) and \ isinstance(other, self.__class__) and \ self._before == other._before and \ self._after == other._after @property - def before(self): + def before(self) -> str: """ The state of task before marked as done. @@ -705,7 +719,7 @@ def before(self): return self._before @property - def after(self): + def after(self) -> str: """ The state of task after marked as done. diff --git a/src/orgparse/extra.py b/src/orgparse/extra.py index 5fefcd6..c720200 100644 --- a/src/orgparse/extra.py +++ b/src/orgparse/extra.py @@ -1,6 +1,8 @@ -import re -from typing import List, Sequence, Dict, Iterator, Iterable, Union, Optional, Type +from __future__ import annotations +import re +from collections.abc import Iterator, Sequence +from typing import Optional, Union RE_TABLE_SEPARATOR = re.compile(r'\s*\|(\-+\+)*\-+\|') RE_TABLE_ROW = re.compile(r'\s*\|([^|]+)+\|') @@ -10,12 +12,12 @@ Row = Sequence[str] class Table: - def __init__(self, lines: List[str]) -> None: + def __init__(self, lines: list[str]) -> None: self._lines = lines @property def blocks(self) -> Iterator[Sequence[Row]]: - group: List[Row] = [] + group: list[Row] = [] first = True for r in self._pre_rows(): if r is None: @@ -49,7 +51,7 @@ def _pre_rows(self) -> Iterator[Optional[Row]]: # TODO use iparse helper? @property - def as_dicts(self) -> 'AsDictHelper': + def as_dicts(self) -> AsDictHelper: bl = list(self.blocks) if len(bl) != 2: raise RuntimeError('Need two-block table to non-ambiguously guess column names') @@ -69,9 +71,9 @@ def __init__(self, columns: Sequence[str], rows: Sequence[Row]) -> None: self.columns = columns self._rows = rows - def __iter__(self) -> Iterator[Dict[str, str]]: + def __iter__(self) -> Iterator[dict[str, str]]: for x in self._rows: - yield {k: v for k, v in zip(self.columns, x)} + yield dict(zip(self.columns, x)) class Gap: @@ -89,8 +91,8 @@ def to_rich_text(text: str) -> Iterator[Rich]: At the moment only tables are supported. ''' lines = text.splitlines(keepends=True) - group: List[str] = [] - last: Type[Rich] = Gap + group: list[str] = [] + last: type[Rich] = Gap def emit() -> Rich: nonlocal group, last if last is Gap: diff --git a/src/orgparse/node.py b/src/orgparse/node.py index 7ed1cdb..f6044a2 100644 --- a/src/orgparse/node.py +++ b/src/orgparse/node.py @@ -1,14 +1,30 @@ -import re -import itertools -from typing import List, Iterable, Iterator, Optional, Union, Tuple, cast, Dict, Set, Sequence, Any +from __future__ import annotations -from .date import OrgDate, OrgDateClock, OrgDateRepeatedTask, parse_sdc, OrgDateScheduled, OrgDateDeadline, OrgDateClosed +import itertools +import re +from collections.abc import Iterable, Iterator, Sequence +from typing import ( + Any, + Optional, + Union, + cast, +) + +from .date import ( + OrgDate, + OrgDateClock, + OrgDateClosed, + OrgDateDeadline, + OrgDateRepeatedTask, + OrgDateScheduled, + parse_sdc, +) +from .extra import Rich, to_rich_text from .inline import to_plain_text -from .extra import to_rich_text, Rich -def lines_to_chunks(lines: Iterable[str]) -> Iterable[List[str]]: - chunk: List[str] = [] +def lines_to_chunks(lines: Iterable[str]) -> Iterable[list[str]]: + chunk: list[str] = [] for l in lines: if RE_NODE_HEADER.search(l): yield chunk @@ -19,7 +35,7 @@ def lines_to_chunks(lines: Iterable[str]) -> Iterable[List[str]]: RE_NODE_HEADER = re.compile(r"^\*+ ") -def parse_heading_level(heading): +def parse_heading_level(heading: str) -> tuple[str, int] | None: """ Get star-stripped heading and its level @@ -32,14 +48,15 @@ def parse_heading_level(heading): >>> parse_heading_level('not heading') # None """ - match = RE_HEADING_STARS.search(heading) - if match: - return (match.group(2), len(match.group(1))) + m = RE_HEADING_STARS.search(heading) + if m is not None: + return (m.group(2), len(m.group(1))) + return None RE_HEADING_STARS = re.compile(r'^(\*+)\s+(.*?)\s*$') -def parse_heading_tags(heading: str) -> Tuple[str, List[str]]: +def parse_heading_tags(heading: str) -> tuple[str, list[str]]: """ Get first tags and heading without tags @@ -74,7 +91,7 @@ def parse_heading_tags(heading: str) -> Tuple[str, List[str]]: RE_HEADING_TAGS = re.compile(r'(.*?)\s*:([\w@:]+):\s*$') -def parse_heading_todos(heading: str, todo_candidates: List[str]) -> Tuple[str, Optional[str]]: +def parse_heading_todos(heading: str, todo_candidates: list[str]) -> tuple[str, Optional[str]]: """ Get TODO keyword and heading without TODO keyword. @@ -116,7 +133,7 @@ def parse_heading_priority(heading): RE_HEADING_PRIORITY = re.compile(r'^\s*\[#([A-Z0-9])\] ?(.*)$') PropertyValue = Union[str, int, float] -def parse_property(line: str) -> Tuple[Optional[str], Optional[PropertyValue]]: +def parse_property(line: str) -> tuple[Optional[str], Optional[PropertyValue]]: """ Get property from given string. @@ -219,7 +236,7 @@ def parse_duration_to_minutes_float(duration: str) -> float: return parse_duration_to_minutes_float(units_part) + parse_duration_to_minutes_float(hms_part) if RE_FLOAT.fullmatch(duration): return float(duration) - raise ValueError("Invalid duration format %s" % duration) + raise ValueError(f"Invalid duration format {duration}") # Conversion factor to minutes for a duration. ORG_DURATION_UNITS = { @@ -231,7 +248,7 @@ def parse_duration_to_minutes_float(duration: str) -> float: "y": 60 * 24 * 365.25, } # Regexp matching for all units. -ORG_DURATION_UNITS_RE = r'(%s)' % r'|'.join(ORG_DURATION_UNITS.keys()) +ORG_DURATION_UNITS_RE = r'({})'.format(r'|'.join(ORG_DURATION_UNITS.keys())) # Regexp matching a duration expressed with H:MM or H:MM:SS format. # Hours can use any number of digits. ORG_DURATION_H_MM_RE = r'[ \t]*[0-9]+(?::[0-9]{2}){1,2}[ \t]*' @@ -244,13 +261,13 @@ def parse_duration_to_minutes_float(duration: str) -> float: RE_ORG_DURATION_UNIT = re.compile(ORG_DURATION_UNIT_RE) # Regexp matching a duration expressed with units. # Allowed units are defined in ORG_DURATION_UNITS. -ORG_DURATION_FULL_RE = r'(?:[ \t]*%s)+[ \t]*' % ORG_DURATION_UNIT_RE +ORG_DURATION_FULL_RE = rf'(?:[ \t]*{ORG_DURATION_UNIT_RE})+[ \t]*' RE_ORG_DURATION_FULL = re.compile(ORG_DURATION_FULL_RE) # Regexp matching a duration expressed with units and H:MM or H:MM:SS format. # Allowed units are defined in ORG_DURATION_UNITS. # Match group A contains units part. # Match group B contains H:MM or H:MM:SS part. -ORG_DURATION_MIXED_RE = r'(?P([ \t]*%s)+)[ \t]*(?P[0-9]+(?::[0-9][0-9]){1,2})[ \t]*' % ORG_DURATION_UNIT_RE +ORG_DURATION_MIXED_RE = rf'(?P([ \t]*{ORG_DURATION_UNIT_RE})+)[ \t]*(?P[0-9]+(?::[0-9][0-9]){{1,2}})[ \t]*' RE_ORG_DURATION_MIXED = re.compile(ORG_DURATION_MIXED_RE) # Regexp matching float numbers. RE_FLOAT = re.compile(r'[0-9]+([.][0-9]*)?') @@ -311,22 +328,30 @@ def parse_seq_todo(line): list(map(strip_fast_access_key, dones.split()))) -class OrgEnv(object): +class OrgEnv: """ Information global to the file (e.g, TODO keywords). """ - def __init__(self, todos=['TODO'], dones=['DONE'], - filename=''): + def __init__( + self, + todos: Sequence[str] | None = None, + dones: Sequence[str] | None = None, + filename: str = '', + ) -> None: + if dones is None: + dones = ['DONE'] + if todos is None: + todos = ['TODO'] self._todos = list(todos) self._dones = list(dones) self._todo_not_specified_in_comment = True self._filename = filename - self._nodes = [] + self._nodes: list[OrgBaseNode] = [] @property - def nodes(self): + def nodes(self) -> list[OrgBaseNode]: """ A list of org nodes. @@ -392,15 +417,12 @@ def all_todo_keys(self): return self._todos + self._dones @property - def filename(self): + def filename(self) -> str: """ Return a path to the source file or similar information. If the org objects are not loaded from a file, this value will be a string of the form ````. - - :rtype: str - """ return self._filename @@ -473,25 +495,18 @@ class OrgBaseNode(Sequence): 5 """ - _body_lines: List[str] # set by the child classes - - def __init__(self, env, index=None) -> None: - """ - Create an :class:`OrgBaseNode` object. - - :type env: :class:`OrgEnv` - :arg env: This will be set to the :attr:`env` attribute. + _body_lines: list[str] # set by the child classes - """ + def __init__(self, env: OrgEnv, index: int | None = None) -> None: self.env = env self.linenumber = cast(int, None) # set in parse_lines # content - self._lines: List[str] = [] + self._lines: list[str] = [] - self._properties: Dict[str, PropertyValue] = {} - self._timestamps: List[OrgDate] = [] + self._properties: dict[str, PropertyValue] = {} + self._timestamps: list[OrgDate] = [] # FIXME: use `index` argument to set index. (Currently it is # done externally in `parse_lines`.) @@ -518,16 +533,14 @@ def __iter__(self): else: break - def __len__(self): + def __len__(self) -> int: return sum(1 for _ in self) - def __nonzero__(self): + def __bool__(self) -> bool: # As self.__len__ returns non-zero value always this is not # needed. This function is only for performance. return True - __bool__ = __nonzero__ # PY3 - def __getitem__(self, key): if isinstance(key, slice): return itertools.islice(self, key.start, key.stop, key.step) @@ -537,22 +550,23 @@ def __getitem__(self, key): for (i, node) in enumerate(self): if i == key: return node - raise IndexError("Out of range {0}".format(key)) + raise IndexError(f"Out of range {key}") else: - raise TypeError("Inappropriate type {0} for {1}" - .format(type(key), type(self))) + raise TypeError(f"Inappropriate type {type(key)} for {type(self)}" + ) # tree structure - def _find_same_level(self, iterable): + def _find_same_level(self, iterable) -> OrgBaseNode | None: for node in iterable: if node.level < self.level: - return + return None if node.level == self.level: return node + return None @property - def previous_same_level(self): + def previous_same_level(self) -> OrgBaseNode | None: """ Return previous node if exists or None otherwise. @@ -574,7 +588,7 @@ def previous_same_level(self): return self._find_same_level(reversed(self.env._nodes[:self._index])) @property - def next_same_level(self): + def next_same_level(self) -> OrgBaseNode | None: """ Return next node if exists or None otherwise. @@ -600,8 +614,9 @@ def _find_parent(self): for node in reversed(self.env._nodes[:self._index]): if node.level < self.level: return node + return None - def get_parent(self, max_level=None): + def get_parent(self, max_level: int | None = None): """ Return a parent node. @@ -751,7 +766,7 @@ def root(self): root = parent @property - def properties(self) -> Dict[str, PropertyValue]: + def properties(self) -> dict[str, PropertyValue]: """ Node properties as a dictionary. @@ -791,7 +806,7 @@ def from_chunk(cls, env, lines): return self def _parse_comments(self): - special_comments: Dict[str, List[str]] = {} + special_comments: dict[str, list[str]] = {} for line in self._lines: parsed = parse_comment(line) if parsed: @@ -825,29 +840,23 @@ def _iparse_properties(self, ilines: Iterator[str]) -> Iterator[str]: # misc @property - def level(self): + def level(self) -> int: """ Level of this node. - - :rtype: int - """ raise NotImplementedError - def _get_tags(self, inher=False) -> Set[str]: + def _get_tags(self, *, inher: bool = False) -> set[str]: # noqa: ARG002 """ Return tags - :arg bool inher: + :arg inher: Mix with tags of all ancestor nodes if ``True``. - - :rtype: set - """ return set() @property - def tags(self) -> Set[str]: + def tags(self) -> set[str]: """ Tags of this and parent's node. @@ -863,7 +872,7 @@ def tags(self) -> Set[str]: return self._get_tags(inher=True) @property - def shallow_tags(self) -> Set[str]: + def shallow_tags(self) -> set[str]: """ Tags defined for this node (don't look-up parent nodes). @@ -879,7 +888,7 @@ def shallow_tags(self) -> Set[str]: return self._get_tags(inher=False) @staticmethod - def _get_text(text, format='plain'): + def _get_text(text, format: str = 'plain'): # noqa: A002 if format == 'plain': return to_plain_text(text) elif format == 'raw': @@ -887,9 +896,9 @@ def _get_text(text, format='plain'): elif format == 'rich': return to_rich_text(text) else: - raise ValueError('format={0} is not supported.'.format(format)) + raise ValueError(f'format={format} is not supported.') - def get_body(self, format='plain') -> str: + def get_body(self, format: str = 'plain') -> str: # noqa: A002 """ Return a string of body text. @@ -928,8 +937,7 @@ def is_root(self): """ return False - def get_timestamps(self, active=False, inactive=False, - range=False, point=False): + def get_timestamps(self, active=False, inactive=False, range=False, point=False): # noqa: FBT002,A002 # will fix later """ Return a list of timestamps in the body text. @@ -1038,14 +1046,14 @@ def __str__(self) -> str: return "\n".join(self._lines) # todo hmm, not sure if it really belongs here and not to OrgRootNode? - def get_file_property_list(self, property): + def get_file_property_list(self, property: str): # noqa: A002 """ Return a list of the selected property """ vals = self._special_comments.get(property.upper(), None) return [] if vals is None else vals - def get_file_property(self, property): + def get_file_property(self, property: str): # noqa: A002 """ Return a single element of the selected property or None if it doesn't exist """ @@ -1055,7 +1063,7 @@ def get_file_property(self, property): elif len(vals) == 1: return vals[0] else: - raise RuntimeError('Multiple values for property {}: {}'.format(property, vals)) + raise RuntimeError(f'Multiple values for property {property}: {vals}') class OrgRootNode(OrgBaseNode): @@ -1071,18 +1079,18 @@ class OrgRootNode(OrgBaseNode): def heading(self) -> str: return '' - def _get_tags(self, inher=False) -> Set[str]: + def _get_tags(self, *, inher: bool = False) -> set[str]: # noqa: ARG002 filetags = self.get_file_property_list('FILETAGS') return set(filetags) @property - def level(self): + def level(self) -> int: return 0 - def get_parent(self, max_level=None): + def get_parent(self, max_level=None): # noqa: ARG002 return None - def is_root(self): + def is_root(self) -> bool: return True # parsers @@ -1111,19 +1119,19 @@ class OrgNode(OrgBaseNode): """ def __init__(self, *args, **kwds) -> None: - super(OrgNode, self).__init__(*args, **kwds) + super().__init__(*args, **kwds) # fixme instead of casts, should organize code in such a way that they aren't necessary self._heading = cast(str, None) - self._level = None - self._tags = cast(List[str], None) + self._level: int | None = None + self._tags = cast(list[str], None) self._todo: Optional[str] = None self._priority = None self._scheduled = OrgDateScheduled(None) self._deadline = OrgDateDeadline(None) self._closed = OrgDateClosed(None) - self._clocklist: List[OrgDateClock] = [] - self._body_lines: List[str] = [] - self._repeated_tasks: List[OrgDateRepeatedTask] = [] + self._clocklist: list[OrgDateClock] = [] + self._body_lines: list[str] = [] + self._repeated_tasks: list[OrgDateRepeatedTask] = [] # parser @@ -1145,10 +1153,11 @@ def _parse_pre(self): def _parse_heading(self) -> None: heading = self._lines[0] - (heading, self._level) = parse_heading_level(heading) + heading_level = parse_heading_level(heading) + if heading_level is not None: + (heading, self._level) = heading_level (heading, self._tags) = parse_heading_tags(heading) - (heading, self._todo) = parse_heading_todos( - heading, self.env.all_todo_keys) + (heading, self._todo) = parse_heading_todos(heading, self.env.all_todo_keys) (heading, self._priority) = parse_heading_priority(heading) self._heading = heading @@ -1218,7 +1227,7 @@ def _iparse_repeated_tasks(self, ilines: Iterator[str]) -> Iterator[str]: \[ (?P [^\]]+) \]''', re.VERBOSE) - def get_heading(self, format='plain'): + def get_heading(self, format: str ='plain') -> str: # noqa: A002 """ Return a string of head text without tags and TODO keywords. @@ -1247,7 +1256,6 @@ def heading(self) -> str: @property def level(self): - return self._level """ Level attribute of this node. Top level node is level 1. @@ -1256,7 +1264,7 @@ def level(self): ... * Node 1 ... ** Node 2 ... ''') - >>> (n1, n2) = root.children + >>> (n1, n2) = list(root[1:]) >>> root.level 0 >>> n1.level @@ -1265,9 +1273,10 @@ def level(self): 2 """ + return self._level @property - def priority(self): + def priority(self) -> str | None: """ Priority attribute of this node. It is None if undefined. @@ -1284,7 +1293,7 @@ def priority(self): """ return self._priority - def _get_tags(self, inher=False) -> Set[str]: + def _get_tags(self, *, inher: bool = False) -> set[str]: tags = set(self._tags) if inher: parent = self.get_parent() diff --git a/src/orgparse/tests/data/00_simple.py b/src/orgparse/tests/data/00_simple.py index c0b23d1..d2de087 100644 --- a/src/orgparse/tests/data/00_simple.py +++ b/src/orgparse/tests/data/00_simple.py @@ -1,17 +1,21 @@ -from typing import Any, Dict, Set +from typing import Any -def nodedict(i, level, todo=None, shallow_tags=set([]), tags=set([])) -> Dict[str, Any]: - return dict( - heading="Heading {0}".format(i), - level=level, - todo=todo, - shallow_tags=shallow_tags, - tags=tags, - ) +def nodedict(i, level, todo=None, shallow_tags=None, tags=None) -> dict[str, Any]: + if tags is None: + tags = set() + if shallow_tags is None: + shallow_tags = set() + return { + "heading": f"Heading {i}", + "level": level, + "todo": todo, + "shallow_tags": shallow_tags, + "tags": tags, + } -def tags(nums) -> Set[str]: +def tags(nums) -> set[str]: return set(map('TAG{0}'.format, nums)) diff --git a/src/orgparse/tests/data/01_attributes.py b/src/orgparse/tests/data/01_attributes.py index d4555de..df720fc 100644 --- a/src/orgparse/tests/data/01_attributes.py +++ b/src/orgparse/tests/data/01_attributes.py @@ -1,73 +1,76 @@ -from typing import Dict, Any +from typing import Any from orgparse.date import ( - OrgDate, OrgDateScheduled, OrgDateDeadline, OrgDateClosed, + OrgDate, OrgDateClock, + OrgDateClosed, + OrgDateDeadline, + OrgDateScheduled, ) -Raw = Dict[str, Any] +Raw = dict[str, Any] -node1: Raw = dict( - heading="A node with a lot of attributes", - priority='A', - scheduled=OrgDateScheduled((2010, 8, 6)), - deadline=OrgDateDeadline((2010, 8, 10)), - closed=OrgDateClosed((2010, 8, 8, 18, 0)), - clock=[ +node1: Raw = { + "heading": "A node with a lot of attributes", + "priority": 'A', + "scheduled": OrgDateScheduled((2010, 8, 6)), + "deadline": OrgDateDeadline((2010, 8, 10)), + "closed": OrgDateClosed((2010, 8, 8, 18, 0)), + "clock": [ OrgDateClock((2010, 8, 8, 17, 40), (2010, 8, 8, 17, 50), 10), OrgDateClock((2010, 8, 8, 17, 00), (2010, 8, 8, 17, 30), 30), ], - properties=dict(Effort=70), - datelist=[OrgDate((2010, 8, 16))], - rangelist=[ + "properties": {"Effort": 70}, + "datelist": [OrgDate((2010, 8, 16))], + "rangelist": [ OrgDate((2010, 8, 7), (2010, 8, 8)), OrgDate((2010, 8, 9, 0, 30), (2010, 8, 10, 13, 20)), OrgDate((2019, 8, 10, 16, 30, 0), (2019, 8, 10, 17, 30, 0)), ], - body="""\ + "body": """\ - <2010-08-16 Mon> DateList - <2010-08-07 Sat>--<2010-08-08 Sun> - <2010-08-09 Mon 00:30>--<2010-08-10 Tue 13:20> RangeList - <2019-08-10 Sat 16:30-17:30> TimeRange""" -) +} -node2: Raw = dict( - heading="A node without any attributed", - priority=None, - scheduled=OrgDateScheduled(None), - deadline=OrgDateDeadline(None), - closed=OrgDateClosed(None), - clock=[], - properties={}, - datelist=[], - rangelist=[], - body="", -) +node2: Raw = { + "heading": "A node without any attributed", + "priority": None, + "scheduled": OrgDateScheduled(None), + "deadline": OrgDateDeadline(None), + "closed": OrgDateClosed(None), + "clock": [], + "properties": {}, + "datelist": [], + "rangelist": [], + "body": "", +} -node3: Raw = dict( - heading="range in deadline", - priority=None, - scheduled=OrgDateScheduled(None), - deadline=OrgDateDeadline((2019, 9, 6, 10, 0), (2019, 9, 6, 11, 20)), - closed=OrgDateClosed(None), - clock=[], - properties={}, - datelist=[], - rangelist=[], - body=" body", -) +node3: Raw = { + "heading": "range in deadline", + "priority": None, + "scheduled": OrgDateScheduled(None), + "deadline": OrgDateDeadline((2019, 9, 6, 10, 0), (2019, 9, 6, 11, 20)), + "closed": OrgDateClosed(None), + "clock": [], + "properties": {}, + "datelist": [], + "rangelist": [], + "body": " body", +} -node4: Raw = dict( - heading="node with a second line but no date", - priority=None, - scheduled=OrgDateScheduled(None), - deadline=OrgDateDeadline(None), - closed=OrgDateClosed(None), - clock=[], - properties={}, - datelist=[], - rangelist=[], - body="body", -) +node4: Raw = { + "heading": "node with a second line but no date", + "priority": None, + "scheduled": OrgDateScheduled(None), + "deadline": OrgDateDeadline(None), + "closed": OrgDateClosed(None), + "clock": [], + "properties": {}, + "datelist": [], + "rangelist": [], + "body": "body", +} data = [node1, node2, node1, node3, node4] diff --git a/src/orgparse/tests/data/02_tree_struct.py b/src/orgparse/tests/data/02_tree_struct.py index 80a8e77..a4ef46c 100644 --- a/src/orgparse/tests/data/02_tree_struct.py +++ b/src/orgparse/tests/data/02_tree_struct.py @@ -1,11 +1,13 @@ -from typing import Any, Dict +from typing import Any -def nodedict(parent, children=[], previous=None, next=None) -> Dict[str, Any]: - return dict(parent_heading=parent, - children_heading=children, - previous_same_level_heading=previous, - next_same_level_heading=next) +def nodedict(parent, children=None, previous=None, next_=None) -> dict[str, Any]: + if children is None: + children = [] + return {'parent_heading': parent, + 'children_heading': children, + 'previous_same_level_heading': previous, + 'next_same_level_heading': next_} data = [nodedict(*args) for args in [ diff --git a/src/orgparse/tests/data/03_repeated_tasks.py b/src/orgparse/tests/data/03_repeated_tasks.py index 18cfe12..fadd5ed 100644 --- a/src/orgparse/tests/data/03_repeated_tasks.py +++ b/src/orgparse/tests/data/03_repeated_tasks.py @@ -1,13 +1,12 @@ -from orgparse.date import OrgDateRepeatedTask, OrgDateDeadline +from orgparse.date import OrgDateDeadline, OrgDateRepeatedTask - -data = [dict( - heading='Pay the rent', - todo='TODO', - deadline=OrgDateDeadline((2005, 10, 1)), - repeated_tasks=[ +data = [{ + 'heading': 'Pay the rent', + 'todo': 'TODO', + 'deadline': OrgDateDeadline((2005, 10, 1)), + 'repeated_tasks': [ OrgDateRepeatedTask((2005, 9, 1, 16, 10, 0), 'TODO', 'DONE'), OrgDateRepeatedTask((2005, 8, 1, 19, 44, 0), 'TODO', 'DONE'), OrgDateRepeatedTask((2005, 7, 1, 17, 27, 0), 'TODO', 'DONE'), ] -)] +}] diff --git a/src/orgparse/tests/data/04_logbook.py b/src/orgparse/tests/data/04_logbook.py index 457c5fa..085b534 100644 --- a/src/orgparse/tests/data/04_logbook.py +++ b/src/orgparse/tests/data/04_logbook.py @@ -1,11 +1,11 @@ from orgparse.date import OrgDateClock -data = [dict( - heading='LOGBOOK drawer test', - clock=[ +data = [{ + 'heading': 'LOGBOOK drawer test', + 'clock': [ OrgDateClock((2012, 10, 26, 16, 1)), OrgDateClock((2012, 10, 26, 14, 50), (2012, 10, 26, 15, 00)), OrgDateClock((2012, 10, 26, 14, 30), (2012, 10, 26, 14, 40)), OrgDateClock((2012, 10, 26, 14, 10), (2012, 10, 26, 14, 20)), ] -)] +}] diff --git a/src/orgparse/tests/data/05_tags.py b/src/orgparse/tests/data/05_tags.py index 52aee63..f4038e8 100644 --- a/src/orgparse/tests/data/05_tags.py +++ b/src/orgparse/tests/data/05_tags.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- def nodedict(i, tags): - return dict( - heading="Node {0}".format(i), - tags=set(tags), - ) + return { + "heading": f"Node {i}", + "tags": set(tags), + } data = [ @@ -17,7 +16,7 @@ def nodedict(i, tags): [["@_"]], [["_tag_"]], ])] + [ - dict(heading='Heading: :with:colon:', tags=set(["tag"])), + {"heading": 'Heading: :with:colon:', "tags": {"tag"}}, ] + [ - dict(heading='unicode', tags=set(['ёж', 'tag', 'háček'])), + {"heading": 'unicode', "tags": {'ёж', 'tag', 'háček'}}, ] diff --git a/src/orgparse/tests/test_data.py b/src/orgparse/tests/test_data.py index 60e4db0..642ee53 100644 --- a/src/orgparse/tests/test_data.py +++ b/src/orgparse/tests/test_data.py @@ -1,12 +1,12 @@ -from glob import glob import os -from pathlib import Path import pickle - -from .. import load, loads +from glob import glob +from pathlib import Path import pytest +from .. import load, loads + DATADIR = os.path.join(os.path.dirname(__file__), 'data') @@ -34,13 +34,13 @@ def value_from_data_key(node, key): if othernode and not othernode.is_root(): return othernode.heading else: - return + return None else: return getattr(node, key) def data_path(dataname, ext): - return os.path.join(DATADIR, '{0}.{1}'.format(dataname, ext)) + return os.path.join(DATADIR, f'{dataname}.{ext}') def get_datanames(): @@ -60,8 +60,8 @@ def test_data(dataname): for (i, (node, kwds)) in enumerate(zip(root[1:], data)): for key in kwds: val = value_from_data_key(node, key) - assert kwds[key] == val, 'check value of {0}-th node of key "{1}" from "{2}".\n\nParsed:\n{3}\n\nReal:\n{4}'.format(i, key, dataname, val, kwds[key]) - assert type(kwds[key]) == type(val), 'check type of {0}-th node of key "{1}" from "{2}".\n\nParsed:\n{3}\n\nReal:\n{4}'.format(i, key, dataname, type(val), type(kwds[key])) + assert kwds[key] == val, f'check value of {i}-th node of key "{key}" from "{dataname}".\n\nParsed:\n{val}\n\nReal:\n{kwds[key]}' + assert type(kwds[key]) == type(val), f'check type of {i}-th node of key "{key}" from "{dataname}".\n\nParsed:\n{type(val)}\n\nReal:\n{type(kwds[key])}' # noqa: E721 assert root.env.filename == oname diff --git a/src/orgparse/tests/test_date.py b/src/orgparse/tests/test_date.py index 0f39575..39de638 100644 --- a/src/orgparse/tests/test_date.py +++ b/src/orgparse/tests/test_date.py @@ -1,6 +1,13 @@ -from orgparse.date import OrgDate, OrgDateScheduled, OrgDateDeadline, OrgDateClock, OrgDateClosed import datetime +from orgparse.date import ( + OrgDate, + OrgDateClock, + OrgDateClosed, + OrgDateDeadline, + OrgDateScheduled, +) + def test_date_as_string() -> None: @@ -39,4 +46,4 @@ def test_date_as_datetime() -> None: testdatetime = (2021, 9, 3, 16, 19, 13) assert OrgDate._as_datetime(datetime.date(*testdate)) == datetime.datetime(*testdate, 0, 0, 0) - assert OrgDate._as_datetime(datetime.datetime(*testdatetime)) == datetime.datetime(*testdatetime) \ No newline at end of file + assert OrgDate._as_datetime(datetime.datetime(*testdatetime)) == datetime.datetime(*testdatetime) diff --git a/src/orgparse/tests/test_hugedata.py b/src/orgparse/tests/test_hugedata.py index f7248ca..aaa7933 100644 --- a/src/orgparse/tests/test_hugedata.py +++ b/src/orgparse/tests/test_hugedata.py @@ -7,10 +7,9 @@ def generate_org_lines(num_top_nodes, depth=3, nodes_per_level=1, _level=1): if depth == 0: return for i in range(num_top_nodes): - yield ("*" * _level) + ' {0}-th heading of level {1}'.format(i, _level) - for child in generate_org_lines( - nodes_per_level, depth - 1, nodes_per_level, _level + 1): - yield child + yield ("*" * _level) + f' {i}-th heading of level {_level}' + yield from generate_org_lines( + nodes_per_level, depth - 1, nodes_per_level, _level + 1) def num_generate_org_lines(num_top_nodes, depth=3, nodes_per_level=1): diff --git a/src/orgparse/tests/test_misc.py b/src/orgparse/tests/test_misc.py index 4cd73e4..5c0b3ff 100644 --- a/src/orgparse/tests/test_misc.py +++ b/src/orgparse/tests/test_misc.py @@ -1,6 +1,7 @@ +from orgparse.date import OrgDate + from .. import load, loads from ..node import OrgEnv -from orgparse.date import OrgDate def test_empty_heading() -> None: @@ -56,7 +57,7 @@ def test_stars(): ** Subheading with text (B1) This subheading is a child of the "anonymous" heading (B), not of heading (A). - """) + """) # noqa: W291 [h1, h2] = root.children assert h1.heading == 'Heading with text (A)' assert h2.heading == '' @@ -95,7 +96,7 @@ def test_add_custom_todo_keys(): todo_keys = ['CUSTOM_TODO'] done_keys = ['CUSTOM_DONE'] filename = '' # default for loads - content = """#+TODO: COMMENT_TODO | COMMENT_DONE + content = """#+TODO: COMMENT_TODO | COMMENT_DONE """ env = OrgEnv(filename=filename) @@ -236,10 +237,10 @@ def test_date_with_cookies() -> None: ('<2007-05-16 Wed 12:30 +1w>', "OrgDate((2007, 5, 16, 12, 30, 0), None, True, ('+', 1, 'w'))"), ] - for (input, expected) in testcases: - root = loads(input) + for (inp, expected) in testcases: + root = loads(inp) output = root[0].datelist[0] - assert str(output) == input + assert str(output) == inp assert repr(output) == expected testcases = [ ('<2006-11-02 Thu 20:00-22:00 +1w>', @@ -247,8 +248,8 @@ def test_date_with_cookies() -> None: ('<2006-11-02 Thu 20:00--22:00 +1w>', "OrgDate((2006, 11, 2, 20, 0, 0), (2006, 11, 2, 22, 0, 0), True, ('+', 1, 'w'))"), ] - for (input, expected) in testcases: - root = loads(input) + for (inp, expected) in testcases: + root = loads(inp) output = root[0].rangelist[0] assert str(output) == "<2006-11-02 Thu 20:00--22:00 +1w>" assert repr(output) == expected @@ -270,8 +271,8 @@ def test_date_with_cookies() -> None: "<2005-10-01 Sat .+1m>", "OrgDateDeadline((2005, 10, 1), None, True, ('.+', 1, 'm'))"), ] - for (input, expected_str, expected_repr) in testcases2: - root = loads(input) + for (inp, expected_str, expected_repr) in testcases2: + root = loads(inp) output = root[1].deadline assert str(output) == expected_str assert repr(output) == expected_repr @@ -292,8 +293,8 @@ def test_date_with_cookies() -> None: "<2005-10-01 Sat .+1m>", "OrgDateScheduled((2005, 10, 1), None, True, ('.+', 1, 'm'))"), ] - for (input, expected_str, expected_repr) in testcases2: - root = loads(input) + for (inp, expected_str, expected_repr) in testcases2: + root = loads(inp) output = root[1].scheduled assert str(output) == expected_str assert repr(output) == expected_repr diff --git a/src/orgparse/tests/test_rich.py b/src/orgparse/tests/test_rich.py index 7fb911b..e423b0d 100644 --- a/src/orgparse/tests/test_rich.py +++ b/src/orgparse/tests/test_rich.py @@ -1,11 +1,11 @@ ''' Tests for rich formatting: tables etc. ''' -from .. import load, loads -from ..extra import Table - import pytest +from .. import loads +from ..extra import Table + def test_table() -> None: root = loads(''' diff --git a/tox.ini b/tox.ini index 681618b..c99ef94 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,7 @@ [tox] minversion = 3.21 # relies on the correct version of Python installed -envlist = tests,mypy -# FIXME will fix ruff in a later commit +envlist = ruff,tests,mypy # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 # hack to prevent .tox from crapping to the project directory toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox From 60c9c27999a4b2903b454b9e8141246debe73aab Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Tue, 20 May 2025 22:32:41 +0100 Subject: [PATCH 87/90] fix pypi publishing --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 359a928..0ae6698 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,3 +53,7 @@ packages = ["src/orgparse"] [tool.hatch.version] source = "vcs" + +[tool.hatch.version.raw-options] +version_scheme = "python-simplified-semver" +local_scheme = "dirty-tag" From 0b945030ba9640596298d4243795902e1b8f0c85 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 30 Oct 2025 22:04:53 +0000 Subject: [PATCH 88/90] chore: bump all ci files/github actions/ruff up to date; enable ty type checking --- .ci/release-uv | 8 +- .github/workflows/main.yml | 54 +++++---- conftest.py | 15 ++- doc/source/conf.py | 51 +++++---- pyproject.toml | 10 +- ruff.toml | 64 +++-------- src/orgparse/__init__.py | 4 +- src/orgparse/date.py | 114 ++++++++++--------- src/orgparse/extra.py | 8 +- src/orgparse/inline.py | 7 +- src/orgparse/node.py | 89 ++++++++------- src/orgparse/tests/data/00_simple.py | 3 +- src/orgparse/tests/data/01_attributes.py | 6 +- src/orgparse/tests/data/02_tree_struct.py | 12 +- src/orgparse/tests/data/03_repeated_tasks.py | 2 +- src/orgparse/tests/data/04_logbook.py | 2 +- src/orgparse/tests/data/05_tags.py | 3 +- src/orgparse/tests/test_data.py | 41 +++---- src/orgparse/tests/test_date.py | 1 - src/orgparse/tests/test_hugedata.py | 7 +- src/orgparse/tests/test_misc.py | 66 ++++++----- src/orgparse/tests/test_rich.py | 6 +- tox.ini | 29 ++++- 23 files changed, 310 insertions(+), 292 deletions(-) diff --git a/.ci/release-uv b/.ci/release-uv index c56697c..4da39b7 100755 --- a/.ci/release-uv +++ b/.ci/release-uv @@ -26,6 +26,7 @@ from subprocess import check_call is_ci = os.environ.get('CI') is not None + def main() -> None: p = argparse.ArgumentParser() p.add_argument('--use-test-pypi', action='store_true') @@ -34,11 +35,7 @@ def main() -> None: publish_url = ['--publish-url', 'https://test.pypi.org/legacy/'] if args.use_test_pypi else [] root = Path(__file__).absolute().parent.parent - os.chdir(root) # just in case - - if is_ci: - # see https://github.com/actions/checkout/issues/217 - check_call('git fetch --prune --unshallow'.split()) + os.chdir(root) # just in case # TODO ok, for now uv won't remove dist dir if it already exists # https://github.com/astral-sh/uv/issues/10293 @@ -46,7 +43,6 @@ def main() -> None: if dist.exists(): shutil.rmtree(dist) - # todo what is --force-pep517? check_call(['uv', 'build']) if not is_ci: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3599e6a..e51bcf6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,10 +6,13 @@ on: branches: '*' tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug: - pull_request: # needed to trigger on others' PRs + + # Needed to trigger on others' PRs. # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them". - workflow_dispatch: # needed to trigger workflows manually - # todo cron? + pull_request: + + # Needed to trigger workflows manually. + workflow_dispatch: inputs: debug_enabled: type: boolean @@ -24,7 +27,7 @@ jobs: fail-fast: false matrix: platform: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] # vvv just an example of excluding stuff from matrix # exclude: [{platform: macos-latest, python-version: '3.6'}] @@ -37,16 +40,16 @@ jobs: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: submodules: recursive fetch-depth: 0 # nicer to have all git history when debugging/for tests - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - - uses: astral-sh/setup-uv@v5 + - uses: astral-sh/setup-uv@v7 with: enable-cache: false # we don't have lock files, so can't use them as cache key @@ -55,44 +58,57 @@ jobs: # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd... - run: bash .ci/run + env: + # only compute lxml coverage on ubuntu; it crashes on windows + CI_MYPY_COVERAGE: ${{ matrix.platform == 'ubuntu-latest' && '--cobertura-xml-report .coverage.mypy' || '' }} - if: matrix.platform == 'ubuntu-latest' # no need to compute coverage for other platforms - uses: actions/upload-artifact@v4 + uses: codecov/codecov-action@v5 with: - include-hidden-files: true - name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }} - path: .coverage.mypy/ + fail_ci_if_error: true # default false + token: ${{ secrets.CODECOV_TOKEN }} + flags: mypy-${{ matrix.python-version }} + files: .coverage.mypy/cobertura.xml pypi: - runs-on: ubuntu-latest + # Do not run it for PRs/cron schedule etc. + # NOTE: release tags are guarded by on: push: tags on the top. + if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/tags/') || (github.event.ref == format('refs/heads/{0}', github.event.repository.master_branch))) + # Ugh, I tried using matrix or something to explicitly generate only test pypi or prod pypi pipelines. + # But github actions is so shit, it's impossible to do any logic at all, e.g. doesn't support conditional matrix, if/else statements for variables etc. + needs: [build] # add all other jobs here + + runs-on: ubuntu-latest + permissions: # necessary for Trusted Publishing id-token: write + steps: # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: submodules: recursive + fetch-depth: 0 # pull all commits to correctly infer vcs version - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.10' - - uses: astral-sh/setup-uv@v5 + - uses: astral-sh/setup-uv@v7 with: enable-cache: false # we don't have lock files, so can't use them as cache key - name: 'release to test pypi' # always deploy merged master to test pypi - if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master' + if: github.event.ref == format('refs/heads/{0}', github.event.repository.master_branch) run: .ci/release-uv --use-test-pypi - - name: 'release to pypi' + - name: 'release to prod pypi' # always deploy tags to release pypi - # NOTE: release tags are guarded by on: push: tags on the top - if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags') + if: startsWith(github.event.ref, 'refs/tags/') run: .ci/release-uv diff --git a/conftest.py b/conftest.py index 91a43a3..627def8 100644 --- a/conftest.py +++ b/conftest.py @@ -20,6 +20,8 @@ # resolve_package_path is called from _pytest.pathlib.import_path # takes a full abs path to the test file and needs to return the path to the 'root' package on the filesystem resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path + + def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]: result = path # search from the test file upwards for parent in result.parents: @@ -30,7 +32,12 @@ def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]: if path.name == 'conftest.py': return resolve_pkg_path_orig(path) raise RuntimeError("Couldn't determine path for ", path) -_pytest.pathlib.resolve_package_path = resolve_package_path + + +# NOTE: seems like it's not necessary anymore? +# keeping it for now just in case +# after https://github.com/pytest-dev/pytest/pull/13426 we should be able to remove the whole conftest +# _pytest.pathlib.resolve_package_path = resolve_package_path # without patching, the orig function returns just a package name for some reason @@ -38,10 +45,14 @@ def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]: # so we need to point it at the absolute path properly # not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure.. search_pypath_orig = _pytest.main.search_pypath + + def search_pypath(module_name: str) -> str: mpath = root_dir / module_name.replace('.', os.sep) if not mpath.is_dir(): mpath = mpath.with_suffix('.py') assert mpath.exists(), mpath # just in case return str(mpath) -_pytest.main.search_pypath = search_pypath + + +_pytest.main.search_pypath = search_pypath # ty: ignore[invalid-assignment] diff --git a/doc/source/conf.py b/doc/source/conf.py index 3f90daa..1e451e7 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- - -from os.path import dirname import sys -sys.path.insert(0, dirname(dirname(dirname(__file__)))) +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # -- General configuration ------------------------------------------------ extensions = [ @@ -19,14 +18,14 @@ import orgparse # General information about the project. -project = u'orgparse' -copyright = u'2012, Takafumi Arakaki' +project = 'orgparse' +copyright = '2012, Takafumi Arakaki' # noqa: A001 # The short X.Y version. # TODO use setup.py for version -version = orgparse.__version__ +version = orgparse.__version__ # ty: ignore[unresolved-attribute] # The full version, including alpha/beta/rc tags. -release = orgparse.__version__ +release = orgparse.__version__ # ty: ignore[unresolved-attribute] exclude_patterns = [] @@ -43,22 +42,19 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto/manual]). latex_documents = [ - ('index', 'orgparse.tex', u'orgparse Documentation', - u'Takafumi Arakaki', 'manual'), + ('index', 'orgparse.tex', 'orgparse Documentation', 'Takafumi Arakaki', 'manual'), ] @@ -66,12 +62,11 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'orgparse', u'orgparse Documentation', - [u'Takafumi Arakaki'], 1) + ('index', 'orgparse', 'orgparse Documentation', ['Takafumi Arakaki'], 1), ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -79,9 +74,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'orgparse', u'orgparse Documentation', - u'Takafumi Arakaki', 'orgparse', 'One line description of project.', - 'Miscellaneous'), + ( + 'index', + 'orgparse', + 'orgparse Documentation', + 'Takafumi Arakaki', + 'orgparse', + 'One line description of project.', + 'Miscellaneous', + ), ] @@ -93,4 +94,4 @@ autodoc_member_order = 'bysource' autodoc_default_flags = ['members'] -inheritance_graph_attrs = dict(rankdir="TB") +inheritance_graph_attrs = {'rankdir': "TB"} diff --git a/pyproject.toml b/pyproject.toml index 0ae6698..0aa9ba0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,17 +32,11 @@ testing = [ "pytest", "ruff", "mypy", - "lxml", # for mypy html coverage + "lxml", # for mypy html coverage + "ty>=0.0.1a25", ] - -# workaround for error during uv publishing -# see https://github.com/astral-sh/uv/issues/9513#issuecomment-2519527822 -[tool.setuptools] -license-files = [] - - [build-system] requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" diff --git a/ruff.toml b/ruff.toml index 0fa381a..e05c3b4 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,37 +1,7 @@ +line-length = 120 # impacts import sorting + lint.extend-select = [ - "F", # flakes rules -- default, but extend just in case - "E", # pycodestyle -- default, but extend just in case - "W", # various warnings - - "B", # 'bugbear' set -- various possible bugs - "C4", # flake8-comprehensions -- unnecessary list/map/dict calls - "COM", # trailing commas - "EXE", # various checks wrt executable files - "I", # sort imports - "ICN", # various import conventions - "FBT", # detect use of boolean arguments - "FURB", # various rules - "PERF", # various potential performance speedups - "PD", # pandas rules - "PIE", # 'misc' lints - "PLC", # pylint convention rules - "PLR", # pylint refactor rules - "PLW", # pylint warnings - "PT", # pytest stuff - "PYI", # various type hinting rules - "RET", # early returns - "RUF", # various ruff-specific rules - "TID", # various imports suggestions - "TRY", # various exception handling rules - "UP", # detect deprecated python stdlib stuff - "FA", # suggest using from __future__ import annotations - # "PTH", # pathlib migration # FIXME do later.. a bit overwhelming - "ARG", # unused argument checks - "A", # builtin shadowing - "G", # logging stuff - # "EM", # TODO hmm could be helpful to prevent duplicate err msg in traceback.. but kinda annoying - - # "ALL", # uncomment this to check for new rules! + "ALL", ] # Preserve types, even if a file imports `from __future__ import annotations` @@ -48,10 +18,10 @@ lint.ignore = [ "FIX", # complains about fixmes/todos -- annoying "TD", # complains about todo formatting -- too annoying "ANN", # missing type annotations? seems way to strict though + "EM" , # suggests assigning all exception messages into a variable first... pretty annoying ### too opinionated style checks "E501", # too long lines - "E702", # Multiple statements on one line (semicolon) "E731", # assigning lambda instead of using def "E741", # Ambiguous variable name: `l` "E742", # Ambiguous class name: `O @@ -66,9 +36,6 @@ lint.ignore = [ ## might be nice .. but later and I don't wanna make it strict "E402", # Module level import not at top of file - "RUF100", # unused noqa -- handle later - "RUF012", # mutable class attrs should be annotated with ClassVar... ugh pretty annoying for user configs - ### these are just nitpicky, we usually know better "PLR0911", # too many return statements "PLR0912", # too many branches @@ -83,10 +50,8 @@ lint.ignore = [ "B009", # calling gettattr with constant attribute -- this is useful to convince mypy "B010", # same as above, but setattr - "B011", # complains about assert False "B017", # pytest.raises(Exception) "B023", # seems to result in false positives? - "B028", # suggest using explicit stacklevel? TODO double check later, but not sure it's useful # complains about useless pass, but has sort of a false positive if the function has a docstring? # this is common for click entrypoints (e.g. in __main__), so disable @@ -106,27 +71,19 @@ lint.ignore = [ "PLW0603", # global variable update.. we usually know why we are doing this "PLW2901", # for loop variable overwritten, usually this is intentional - "PT011", # pytest raises should is too broad - "PT012", # pytest raises should contain a single statement + "PT011", # pytest raises is too broad "COM812", # trailing comma missing -- mostly just being annoying with long multiline strings - "PD901", # generic variable name df - "TRY003", # suggests defining exception messages in exception class -- kinda annoying - "TRY004", # prefer TypeError -- don't see the point "TRY201", # raise without specifying exception name -- sometimes hurts readability - "TRY400", # TODO double check this, might be useful + "TRY400", # a bit dumb, and results in false positives (see https://github.com/astral-sh/ruff/issues/18070) "TRY401", # redundant exception in logging.exception call? TODO double check, might result in excessive logging - "PGH", # TODO force error code in mypy instead? although it also has blanket noqa rule - "TID252", # Prefer absolute imports over relative imports from parent modules - "UP038", # suggests using | (union) in isisntance checks.. but it results in slower code - ## too annoying - "T20", # just complains about prints and pprints + "T20", # just complains about prints and pprints (TODO maybe consider later?) "Q", # flake quotes, too annoying "C90", # some complexity checking "G004", # logging statement uses f string @@ -134,7 +91,12 @@ lint.ignore = [ "SLF001", # private member accessed "BLE001", # do not catch 'blind' Exception "INP001", # complains about implicit namespace packages - "SIM", # some if statements crap + "SIM102", # if statements collapsing, often hurts readability + "SIM103", # multiple conditions collapsing, often hurts readability + "SIM105", # suggests using contextlib.suppress instad of try/except -- this wouldn't be mypy friendly + "SIM108", # suggests using ternary operation instead of if -- hurts readability + "SIM110", # suggests using any(...) instead of for look/return -- hurts readability + "SIM117", # suggests using single with statement instead of nested -- doesn't work in tests "RSE102", # complains about missing parens in exceptions ## ] diff --git a/src/orgparse/__init__.py b/src/orgparse/__init__.py index d699f4f..110c474 100644 --- a/src/orgparse/__init__.py +++ b/src/orgparse/__init__.py @@ -145,7 +145,7 @@ def load(path: Union[str, Path, TextIO], env: Optional[OrgEnv] = None) -> OrgNod return loadi(all_lines, filename=filename, env=env) -def loads(string: str, filename: str='', env: Optional[OrgEnv]=None) -> OrgNode: +def loads(string: str, filename: str = '', env: Optional[OrgEnv] = None) -> OrgNode: """ Load org-mode document from a string. @@ -155,7 +155,7 @@ def loads(string: str, filename: str='', env: Optional[OrgEnv]=None) -> return loadi(string.splitlines(), filename=filename, env=env) -def loadi(lines: Iterable[str], filename: str='', env: Optional[OrgEnv]=None) -> OrgNode: +def loadi(lines: Iterable[str], filename: str = '', env: Optional[OrgEnv] = None) -> OrgNode: """ Load org-mode document from an iterative object. diff --git a/src/orgparse/date.py b/src/orgparse/date.py index ccaa5c3..1685f32 100644 --- a/src/orgparse/date.py +++ b/src/orgparse/date.py @@ -10,8 +10,7 @@ def total_seconds(td: timedelta) -> float: """Equivalent to `datetime.timedelta.total_seconds`.""" - return float(td.microseconds + - (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10 ** 6 + return float(td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 def total_minutes(td: timedelta) -> float: @@ -126,13 +125,13 @@ def gene_timestamp_regex(brtype: str, prefix: str | None = None, *, nocookie: bo (?P<{prefix}warndwmy> [hdwmy]) )? """ - # http://www.pythonregex.com/ regex = ''.join([ bo, regex_date_time, regex_cookie if nocookie or brtype != 'nobrace' else '', '({ignore}*?)', - bc]) + bc, + ]) # fmt: skip return regex.format(prefix=prefix, ignore=ignore) @@ -155,24 +154,27 @@ def is_same_day(date0, date1) -> bool: """ Check if two dates or datetimes are on the same day """ - return (OrgDate._date_to_tuple(date0)[:3] == OrgDate._date_to_tuple(date1)[:3]) + return OrgDate._date_to_tuple(date0)[:3] == OrgDate._date_to_tuple(date1)[:3] TIMESTAMP_NOBRACE_RE = re.compile( gene_timestamp_regex('nobrace', prefix=''), - re.VERBOSE) + re.VERBOSE, +) TIMESTAMP_RE = re.compile( - '|'.join((gene_timestamp_regex('active'), - gene_timestamp_regex('inactive'))), - re.VERBOSE) + '|'.join(( + gene_timestamp_regex('active'), + gene_timestamp_regex('inactive'), + )), + re.VERBOSE, +) # fmt: skip _Repeater = tuple[str, int, str] class OrgDate: - _active_default = True """ The default active value. @@ -194,7 +196,7 @@ def __init__( self, start, end=None, - active: bool | None = None, + active: bool | None = None, # noqa: FBT001 repeater: _Repeater | None = None, warning: _Repeater | None = None, ) -> None: @@ -255,7 +257,7 @@ def _to_date(date) -> DateIsh: "Automatic conversion to the datetime object " "requires at least 3 elements in the tuple. " f"Only {len(date)} elements are in the given tuple '{date}'." - ) + ) elif isinstance(date, (int, float)): return datetime.datetime.fromtimestamp(date) else: @@ -268,7 +270,7 @@ def _date_to_tuple(date: DateIsh) -> tuple[int, ...]: elif isinstance(date, datetime.date): return tuple(date.timetuple()[:3]) else: - raise RuntimeError(f"can't happen: {date}") + raise TypeError(f"can't happen: {date} {type(date)}") def __repr__(self) -> str: args = [ @@ -312,15 +314,18 @@ def __str__(self) -> str: def __bool__(self) -> bool: return bool(self._start) + def __hash__(self) -> int: + return hash((self._start, self._end, self._active, self._repeater, self._warning)) + def __eq__(self, other) -> bool: - if (isinstance(other, OrgDate) and - self._start is None and - other._start is None): + if isinstance(other, OrgDate) and self._start is None and other._start is None: return True - return (isinstance(other, self.__class__) and - self._start == other._start and - self._end == other._end and - self._active == other._active) + return ( + isinstance(other, self.__class__) + and self._start == other._start + and self._end == other._end + and self._active == other._active + ) @property def start(self) -> DateIsh: @@ -390,11 +395,10 @@ def has_overlap(self, other) -> bool: if not isinstance(other, OrgDate): other = OrgDate(other) if self.has_end(): - return (self._datetime_in_range(other.start) or - self._datetime_in_range(other.end)) + return self._datetime_in_range(other.start) or self._datetime_in_range(other.end) elif other.has_end(): return other._datetime_in_range(self.start) - elif self.start == other.get_start: + elif self.start == other.start: return True else: return False @@ -419,11 +423,11 @@ def _as_datetime(date) -> datetime.datetime: @staticmethod def _daterange_from_groupdict(dct, prefix='') -> tuple[list, Optional[list]]: - start_keys = ['year', 'month', 'day', 'hour' , 'min'] - end_keys = ['year', 'month', 'day', 'end_hour', 'end_min'] + start_keys = ['year', 'month', 'day', 'hour' , 'min'] # fmt: skip + end_keys = ['year', 'month', 'day', 'end_hour', 'end_min'] # fmt: skip start_range = list(map(int, filter(None, (dct[prefix + k] for k in start_keys)))) end_range: Optional[list] - end_range = list(map(int, filter(None, (dct[prefix + k] for k in end_keys)))) + end_range = list(map(int, filter(None, (dct[prefix + k] for k in end_keys)))) if len(end_range) < len(end_keys): end_range = None return (start_range, end_range) @@ -451,7 +455,7 @@ def list_from_str(cls, string: str) -> list[OrgDate]: cookie_suffix = ['pre', 'num', 'dwmy'] match = TIMESTAMP_RE.search(string) if match: - rest = string[match.end():] + rest = string[match.end() :] mdict = match.groupdict() if mdict['active_year']: prefix = 'active_' @@ -474,17 +478,20 @@ def list_from_str(cls, string: str) -> list[OrgDate]: has_rangedash = rest.startswith(rangedash) match2 = TIMESTAMP_RE.search(rest) if has_rangedash else None if has_rangedash and match2: - rest = rest[match2.end():] + rest = rest[match2.end() :] # no need for check activeness here because of the rangedash mdict2 = match2.groupdict() odate = cls( cls._datetuple_from_groupdict(mdict, prefix), cls._datetuple_from_groupdict(mdict2, prefix), - active=active, repeater=repeater, warning=warning) + active=active, + repeater=repeater, + warning=warning, + ) else: odate = cls( - *cls._daterange_from_groupdict(mdict, prefix), - active=active, repeater=repeater, warning=warning) + *cls._daterange_from_groupdict(mdict, prefix), active=active, repeater=repeater, warning=warning + ) return [odate, *cls.list_from_str(rest)] else: return [] @@ -503,8 +510,7 @@ def from_str(cls, string: str) -> OrgDate: match = cls._from_str_re.match(string) if match: mdict = match.groupdict() - return cls(cls._datetuple_from_groupdict(mdict), - active=cls._active_default) + return cls(cls._datetuple_from_groupdict(mdict), active=cls._active_default) else: return cls(None) @@ -516,12 +522,13 @@ def compile_sdc_re(sdctype): return re.compile( r'^(?!\#).*{}:\s+{}'.format( sdctype, - gene_timestamp_regex(brtype, prefix='', nocookie=True)), - re.VERBOSE) + gene_timestamp_regex(brtype, prefix='', nocookie=True), + ), + re.VERBOSE, + ) class OrgDateSDCBase(OrgDate): - _re = None # override this! # FIXME: use OrgDate.from_str @@ -535,7 +542,7 @@ def from_str(cls, string): start = cls._datetuple_from_groupdict(mdict) end = None end_hour = mdict['end_hour'] - end_min = mdict['end_min'] + end_min = mdict['end_min'] if end_hour is not None and end_min is not None: end_dict = {} end_dict.update(mdict) @@ -553,38 +560,37 @@ def from_str(cls, string): keys = [prefix + 'warn' + suffix for suffix in cookie_suffix] values = [mdict[k] for k in keys] warning = (values[0], int(values[1]), values[2]) - return cls(start, end, active=cls._active_default, - repeater=repeater, warning=warning) + return cls(start, end, active=cls._active_default, repeater=repeater, warning=warning) else: return cls(None) class OrgDateScheduled(OrgDateSDCBase): """Date object to represent SCHEDULED attribute.""" + _re = compile_sdc_re('SCHEDULED') _active_default = True class OrgDateDeadline(OrgDateSDCBase): """Date object to represent DEADLINE attribute.""" + _re = compile_sdc_re('DEADLINE') _active_default = True class OrgDateClosed(OrgDateSDCBase): """Date object to represent CLOSED attribute.""" + _re = compile_sdc_re('CLOSED') _active_default = False def parse_sdc(string): - return (OrgDateScheduled.from_str(string), - OrgDateDeadline.from_str(string), - OrgDateClosed.from_str(string)) + return (OrgDateScheduled.from_str(string), OrgDateDeadline.from_str(string), OrgDateClosed.from_str(string)) class OrgDateClock(OrgDate): - """ Date object to represent CLOCK attributes. @@ -635,8 +641,7 @@ def is_duration_consistent(self): False """ - return (self._duration is None or - self._duration == total_minutes(self.duration)) + return self._duration is None or self._duration == total_minutes(self.duration) @classmethod def from_str(cls, line: str) -> OrgDateClock: @@ -677,11 +682,10 @@ def from_str(cls, line: str) -> OrgDateClock: r'^(?!#).*CLOCK:\s+' r'\[(\d+)\-(\d+)\-(\d+)[^\]\d]*(\d+)\:(\d+)\]' r'(--\[(\d+)\-(\d+)\-(\d+)[^\]\d]*(\d+)\:(\d+)\]\s+=>\s+(\d+)\:(\d+))?' - ) + ) class OrgDateRepeatedTask(OrgDate): - """ Date object to represent repeated tasks. """ @@ -697,14 +701,18 @@ def __repr__(self) -> str: args: list = [self._date_to_tuple(self.start), self.before, self.after] if self._active is not self._active_default: args.append(self._active) - return '{}({})'.format( - self.__class__.__name__, ', '.join(map(repr, args))) + return '{}({})'.format(self.__class__.__name__, ', '.join(map(repr, args))) + + def __hash__(self) -> int: + return hash((self._before, self._after)) def __eq__(self, other) -> bool: - return super().__eq__(other) and \ - isinstance(other, self.__class__) and \ - self._before == other._before and \ - self._after == other._after + return ( + super().__eq__(other) + and isinstance(other, self.__class__) + and self._before == other._before + and self._after == other._after + ) @property def before(self) -> str: diff --git a/src/orgparse/extra.py b/src/orgparse/extra.py index c720200..e89343e 100644 --- a/src/orgparse/extra.py +++ b/src/orgparse/extra.py @@ -11,6 +11,7 @@ Row = Sequence[str] + class Table: def __init__(self, lines: list[str]) -> None: self._lines = lines @@ -82,6 +83,8 @@ class Gap: Rich = Union[Table, Gap] + + def to_rich_text(text: str) -> Iterator[Rich]: ''' Convert an org-mode text into a 'rich' text, e.g. tables/lists/etc, interleaved by gaps. @@ -93,12 +96,13 @@ def to_rich_text(text: str) -> Iterator[Rich]: lines = text.splitlines(keepends=True) group: list[str] = [] last: type[Rich] = Gap + def emit() -> Rich: nonlocal group, last - if last is Gap: + if last is Gap: res = Gap() elif last is Table: - res = Table(group) # type: ignore[assignment] + res = Table(group) # type: ignore[assignment] else: raise RuntimeError(f'Unexpected type {last}') group = [] diff --git a/src/orgparse/inline.py b/src/orgparse/inline.py index 043c99d..a2057fc 100644 --- a/src/orgparse/inline.py +++ b/src/orgparse/inline.py @@ -25,9 +25,7 @@ def to_plain_text(org_text): See also: info:org#Link format """ - return RE_LINK.sub( - lambda m: m.group('desc0') or m.group('desc1'), - org_text) + return RE_LINK.sub(lambda m: m.group('desc0') or m.group('desc1'), org_text) RE_LINK = re.compile( @@ -45,4 +43,5 @@ def to_plain_text(org_text): \] \] ) """, - re.VERBOSE) + re.VERBOSE, +) diff --git a/src/orgparse/node.py b/src/orgparse/node.py index f6044a2..5794b43 100644 --- a/src/orgparse/node.py +++ b/src/orgparse/node.py @@ -32,6 +32,7 @@ def lines_to_chunks(lines: Iterable[str]) -> Iterable[list[str]]: chunk.append(l) yield chunk + RE_NODE_HEADER = re.compile(r"^\*+ ") @@ -53,6 +54,7 @@ def parse_heading_level(heading: str) -> tuple[str, int] | None: return (m.group(2), len(m.group(1))) return None + RE_HEADING_STARS = re.compile(r'^(\*+)\s+(.*?)\s*$') @@ -87,6 +89,7 @@ def parse_heading_tags(heading: str) -> tuple[str, list[str]]: tags = [] return (heading, tags) + # Tags are normal words containing letters, numbers, '_', and '@'. https://orgmode.org/manual/Tags.html RE_HEADING_TAGS = re.compile(r'(.*?)\s*:([\w@:]+):\s*$') @@ -106,7 +109,7 @@ def parse_heading_todos(heading: str, todo_candidates: list[str]) -> tuple[str, if heading == todo: return ('', todo) if heading.startswith(todo + ' '): - return (heading[len(todo) + 1:], todo) + return (heading[len(todo) + 1 :], todo) return (heading, None) @@ -130,9 +133,12 @@ def parse_heading_priority(heading): else: return (heading, None) + RE_HEADING_PRIORITY = re.compile(r'^\s*\[#([A-Z0-9])\] ?(.*)$') PropertyValue = Union[str, int, float] + + def parse_property(line: str) -> tuple[Optional[str], Optional[PropertyValue]]: """ Get property from given string. @@ -153,8 +159,10 @@ def parse_property(line: str) -> tuple[Optional[str], Optional[PropertyValue]]: prop_val = parse_duration_to_minutes(prop_val) return (prop_key, prop_val) + RE_PROP = re.compile(r'^\s*:(.*?):\s*(.*?)\s*$') + def parse_duration_to_minutes(duration: str) -> Union[float, int]: """ Parse duration minutes from given string. @@ -185,6 +193,7 @@ def parse_duration_to_minutes(duration: str) -> Union[float, int]: minutes = parse_duration_to_minutes_float(duration) return int(minutes) if minutes.is_integer() else minutes + def parse_duration_to_minutes_float(duration: str) -> float: """ Parse duration minutes from given string. @@ -238,6 +247,7 @@ def parse_duration_to_minutes_float(duration: str) -> float: return float(duration) raise ValueError(f"Invalid duration format {duration}") + # Conversion factor to minutes for a duration. ORG_DURATION_UNITS = { "min": 1, @@ -272,7 +282,9 @@ def parse_duration_to_minutes_float(duration: str) -> float: # Regexp matching float numbers. RE_FLOAT = re.compile(r'[0-9]+([.][0-9]*)?') -def parse_comment(line: str): # -> Optional[Tuple[str, Sequence[str]]]: # todo wtf?? it says 'ABCMeta isn't subscriptable??' + +# -> Optional[Tuple[str, Sequence[str]]]: # todo wtf?? it says 'ABCMeta isn't subscriptable??' +def parse_comment(line: str): """ Parse special comment such as ``#+SEQ_TODO`` @@ -288,7 +300,7 @@ def parse_comment(line: str): # -> Optional[Tuple[str, Sequence[str]]]: # todo end = match.end(0) comment = line[end:].split(':', maxsplit=1) if len(comment) >= 2: - key = comment[0] + key = comment[0] value = comment[1].strip() if key.upper() == 'FILETAGS': # just legacy behaviour; it seems like filetags is the only one that separated by ':' @@ -324,12 +336,13 @@ def parse_seq_todo(line): else: (todos, dones) = (line, '') strip_fast_access_key = lambda x: x.split('(', 1)[0] - return (list(map(strip_fast_access_key, todos.split())), - list(map(strip_fast_access_key, dones.split()))) + return ( + list(map(strip_fast_access_key, todos.split())), + list(map(strip_fast_access_key, dones.split())), + ) class OrgEnv: - """ Information global to the file (e.g, TODO keywords). """ @@ -435,7 +448,6 @@ def from_chunks(self, chunks): class OrgBaseNode(Sequence): - """ Base class for :class:`OrgRootNode` and :class:`OrgNode` @@ -495,12 +507,12 @@ class OrgBaseNode(Sequence): 5 """ - _body_lines: list[str] # set by the child classes + _body_lines: list[str] # set by the child classes def __init__(self, env: OrgEnv, index: int | None = None) -> None: self.env = env - self.linenumber = cast(int, None) # set in parse_lines + self.linenumber = cast(int, None) # set in parse_lines # content self._lines: list[str] = [] @@ -527,7 +539,7 @@ def __init__(self, env: OrgEnv, index: int | None = None) -> None: def __iter__(self): yield self level = self.level - for node in self.env._nodes[self._index + 1:]: + for node in self.env._nodes[self._index + 1 :]: if node.level > level: yield node else: @@ -547,13 +559,12 @@ def __getitem__(self, key): elif isinstance(key, int): if key < 0: key += len(self) - for (i, node) in enumerate(self): + for i, node in enumerate(self): if i == key: return node raise IndexError(f"Out of range {key}") else: - raise TypeError(f"Inappropriate type {type(key)} for {type(self)}" - ) + raise TypeError(f"Inappropriate type {type(key)} for {type(self)}") # tree structure @@ -585,7 +596,7 @@ def previous_same_level(self) -> OrgBaseNode | None: True """ - return self._find_same_level(reversed(self.env._nodes[:self._index])) + return self._find_same_level(reversed(self.env._nodes[: self._index])) @property def next_same_level(self) -> OrgBaseNode | None: @@ -607,11 +618,11 @@ def next_same_level(self) -> OrgBaseNode | None: True """ - return self._find_same_level(self.env._nodes[self._index + 1:]) + return self._find_same_level(self.env._nodes[self._index + 1 :]) # FIXME: cache parent node def _find_parent(self): - for node in reversed(self.env._nodes[:self._index]): + for node in reversed(self.env._nodes[: self._index]): if node.level < self.level: return node return None @@ -702,7 +713,7 @@ def parent(self): # FIXME: cache children nodes def _find_children(self): - nodeiter = iter(self.env._nodes[self._index + 1:]) + nodeiter = iter(self.env._nodes[self._index + 1 :]) try: node = next(nodeiter) except StopIteration: @@ -811,7 +822,7 @@ def _parse_comments(self): parsed = parse_comment(line) if parsed: (key, vals) = parsed - key = key.upper() # case insensitive, so keep as uppercase + key = key.upper() # case insensitive, so keep as uppercase special_comments.setdefault(key, []).extend(vals) self._special_comments = special_comments # parse TODO keys and store in OrgEnv @@ -905,8 +916,7 @@ def get_body(self, format: str = 'plain') -> str: # noqa: A002 See also: :meth:`get_heading`. """ - return self._get_text( - '\n'.join(self._body_lines), format) if self._lines else '' + return self._get_text('\n'.join(self._body_lines), format) if self._lines else '' @property def body(self) -> str: @@ -916,7 +926,7 @@ def body(self) -> str: @property def body_rich(self) -> Iterator[Rich]: r = self.get_body(format='rich') - return cast(Iterator[Rich], r) # meh.. + return cast(Iterator[Rich], r) # meh.. @property def heading(self) -> str: @@ -990,11 +1000,13 @@ def get_timestamps(self, active=False, inactive=False, range=False, point=False) """ return [ - ts for ts in self._timestamps if - (((active and ts.is_active()) or - (inactive and not ts.is_active())) and - ((range and ts.has_end()) or - (point and not ts.has_end())))] + ts + for ts in self._timestamps + if ( + ((active and ts.is_active()) or (inactive and not ts.is_active())) + and ((range and ts.has_end()) or (point and not ts.has_end())) + ) + ] @property def datelist(self): @@ -1067,7 +1079,6 @@ def get_file_property(self, property: str): # noqa: A002 class OrgRootNode(OrgBaseNode): - """ Node to represent a file. Its body contains all lines before the first headline @@ -1110,7 +1121,6 @@ def _iparse_timestamps(self, ilines: Iterator[str]) -> Iterator[str]: class OrgNode(OrgBaseNode): - """ Node to represent normal org node @@ -1141,7 +1151,7 @@ def _parse_pre(self): # FIXME: make the following parsers "lazy" ilines: Iterator[str] = iter(self._lines) try: - next(ilines) # skip heading + next(ilines) # skip heading except StopIteration: return ilines = self._iparse_sdc(ilines) @@ -1180,9 +1190,7 @@ def _iparse_sdc(self, ilines: Iterator[str]) -> Iterator[str]: return (self._scheduled, self._deadline, self._closed) = parse_sdc(line) - if not (self._scheduled or - self._deadline or - self._closed): + if not (self._scheduled or self._deadline or self._closed): yield line # when none of them were found for line in ilines: @@ -1214,8 +1222,7 @@ def _iparse_repeated_tasks(self, ilines: Iterator[str]) -> Iterator[str]: done_state = mdict['done'] todo_state = mdict['todo'] date = OrgDate.from_str(mdict['date']) - self._repeated_tasks.append( - OrgDateRepeatedTask(date.start, todo_state, done_state)) + self._repeated_tasks.append(OrgDateRepeatedTask(date.start, todo_state, done_state)) else: yield line @@ -1225,9 +1232,10 @@ def _iparse_repeated_tasks(self, ilines: Iterator[str]) -> Iterator[str]: State \s+ "(?P [^"]+)" \s+ from \s+ "(?P [^"]+)" \s+ \[ (?P [^\]]+) \]''', - re.VERBOSE) + re.VERBOSE, + ) - def get_heading(self, format: str ='plain') -> str: # noqa: A002 + def get_heading(self, format: str = 'plain') -> str: # noqa: A002 """ Return a string of head text without tags and TODO keywords. @@ -1390,10 +1398,7 @@ def has_date(self): """ Return ``True`` if it has any kind of timestamp """ - return (self.scheduled or - self.deadline or - self.datelist or - self.rangelist) + return self.scheduled or self.deadline or self.datelist or self.rangelist @property def repeated_tasks(self): @@ -1454,14 +1459,14 @@ def parse_lines(lines: Iterable[str], filename, env=None) -> OrgNode: nodes = env.from_chunks(ch2) nodelist = [] for lineno, node in zip(linenos, nodes): - lineno += 1 # in text editors lines are 1-indexed + lineno += 1 # in text editors lines are 1-indexed node.linenumber = lineno nodelist.append(node) # parse headings (level, TODO, TAGs, and heading) nodelist[0]._index = 0 # parse the root node nodelist[0]._parse_pre() - for (i, node) in enumerate(nodelist[1:], 1): # nodes except root node + for i, node in enumerate(nodelist[1:], 1): # nodes except root node node._index = i node._parse_pre() env._nodes = nodelist diff --git a/src/orgparse/tests/data/00_simple.py b/src/orgparse/tests/data/00_simple.py index d2de087..23ad86c 100644 --- a/src/orgparse/tests/data/00_simple.py +++ b/src/orgparse/tests/data/00_simple.py @@ -34,4 +34,5 @@ def tags(nums) -> set[str]: [4, None , tags([1]) , tags([1, 2]) ], [2, None , tags([]) , tags([2]) ], [1], - ])] + ]) +] # fmt: skip diff --git a/src/orgparse/tests/data/01_attributes.py b/src/orgparse/tests/data/01_attributes.py index df720fc..467df02 100644 --- a/src/orgparse/tests/data/01_attributes.py +++ b/src/orgparse/tests/data/01_attributes.py @@ -19,19 +19,19 @@ "clock": [ OrgDateClock((2010, 8, 8, 17, 40), (2010, 8, 8, 17, 50), 10), OrgDateClock((2010, 8, 8, 17, 00), (2010, 8, 8, 17, 30), 30), - ], + ], "properties": {"Effort": 70}, "datelist": [OrgDate((2010, 8, 16))], "rangelist": [ OrgDate((2010, 8, 7), (2010, 8, 8)), OrgDate((2010, 8, 9, 0, 30), (2010, 8, 10, 13, 20)), OrgDate((2019, 8, 10, 16, 30, 0), (2019, 8, 10, 17, 30, 0)), - ], + ], "body": """\ - <2010-08-16 Mon> DateList - <2010-08-07 Sat>--<2010-08-08 Sun> - <2010-08-09 Mon 00:30>--<2010-08-10 Tue 13:20> RangeList - - <2019-08-10 Sat 16:30-17:30> TimeRange""" + - <2019-08-10 Sat 16:30-17:30> TimeRange""", } node2: Raw = { diff --git a/src/orgparse/tests/data/02_tree_struct.py b/src/orgparse/tests/data/02_tree_struct.py index a4ef46c..86b6314 100644 --- a/src/orgparse/tests/data/02_tree_struct.py +++ b/src/orgparse/tests/data/02_tree_struct.py @@ -4,10 +4,12 @@ def nodedict(parent, children=None, previous=None, next_=None) -> dict[str, Any]: if children is None: children = [] - return {'parent_heading': parent, - 'children_heading': children, - 'previous_same_level_heading': previous, - 'next_same_level_heading': next_} + return { + 'parent_heading': parent, + 'children_heading': children, + 'previous_same_level_heading': previous, + 'next_same_level_heading': next_, + } data = [nodedict(*args) for args in [ @@ -43,4 +45,4 @@ def nodedict(parent, children=None, previous=None, next_=None) -> dict[str, Any] ('G6-H1', [], 'G6-H2'), # G7 (None, [], 'G6-H1'), -]] +]] # fmt: skip diff --git a/src/orgparse/tests/data/03_repeated_tasks.py b/src/orgparse/tests/data/03_repeated_tasks.py index fadd5ed..17336e0 100644 --- a/src/orgparse/tests/data/03_repeated_tasks.py +++ b/src/orgparse/tests/data/03_repeated_tasks.py @@ -9,4 +9,4 @@ OrgDateRepeatedTask((2005, 8, 1, 19, 44, 0), 'TODO', 'DONE'), OrgDateRepeatedTask((2005, 7, 1, 17, 27, 0), 'TODO', 'DONE'), ] -}] +}] # fmt: skip diff --git a/src/orgparse/tests/data/04_logbook.py b/src/orgparse/tests/data/04_logbook.py index 085b534..2443683 100644 --- a/src/orgparse/tests/data/04_logbook.py +++ b/src/orgparse/tests/data/04_logbook.py @@ -8,4 +8,4 @@ OrgDateClock((2012, 10, 26, 14, 30), (2012, 10, 26, 14, 40)), OrgDateClock((2012, 10, 26, 14, 10), (2012, 10, 26, 14, 20)), ] -}] +}] # fmt: skip diff --git a/src/orgparse/tests/data/05_tags.py b/src/orgparse/tests/data/05_tags.py index f4038e8..19447f4 100644 --- a/src/orgparse/tests/data/05_tags.py +++ b/src/orgparse/tests/data/05_tags.py @@ -1,4 +1,3 @@ - def nodedict(i, tags): return { "heading": f"Node {i}", @@ -19,4 +18,4 @@ def nodedict(i, tags): {"heading": 'Heading: :with:colon:', "tags": {"tag"}}, ] + [ {"heading": 'unicode', "tags": {'ёж', 'tag', 'háček'}}, - ] + ] # fmt: skip diff --git a/src/orgparse/tests/test_data.py b/src/orgparse/tests/test_data.py index 642ee53..c271273 100644 --- a/src/orgparse/tests/test_data.py +++ b/src/orgparse/tests/test_data.py @@ -1,20 +1,19 @@ -import os import pickle -from glob import glob +from collections.abc import Iterator from pathlib import Path import pytest from .. import load, loads -DATADIR = os.path.join(os.path.dirname(__file__), 'data') +DATADIR = Path(__file__).parent / 'data' -def load_data(path): +def load_data(path: Path): """Load data from python file""" ns = {} # type: ignore[var-annotated] # read_bytes() and compile hackery to avoid encoding issues (e.g. see 05_tags) - exec(compile(Path(path).read_bytes(), path, 'exec'), ns) + exec(compile(path.read_bytes(), path, 'exec'), ns) return ns['data'] @@ -26,10 +25,11 @@ def value_from_data_key(node, key): return node.tags elif key == 'children_heading': return [c.heading for c in node.children] - elif key in ('parent_heading', - 'previous_same_level_heading', - 'next_same_level_heading', - ): + elif key in ( + 'parent_heading', + 'previous_same_level_heading', + 'next_same_level_heading', + ): othernode = getattr(node, key.rsplit('_', 1)[0]) if othernode and not othernode.is_root(): return othernode.heading @@ -39,13 +39,13 @@ def value_from_data_key(node, key): return getattr(node, key) -def data_path(dataname, ext): - return os.path.join(DATADIR, f'{dataname}.{ext}') +def data_path(dataname: str, ext: str) -> Path: + return DATADIR / f'{dataname}.{ext}' -def get_datanames(): - for oname in sorted(glob(os.path.join(DATADIR, '*.org'))): - yield os.path.splitext(os.path.basename(oname))[0] +def get_datanames() -> Iterator[str]: + for oname in sorted(DATADIR.glob('*.org')): + yield oname.stem @pytest.mark.parametrize('dataname', get_datanames()) @@ -57,13 +57,17 @@ def test_data(dataname): data = load_data(data_path(dataname, "py")) root = load(oname) - for (i, (node, kwds)) in enumerate(zip(root[1:], data)): + for i, (node, kwds) in enumerate(zip(root[1:], data)): for key in kwds: val = value_from_data_key(node, key) - assert kwds[key] == val, f'check value of {i}-th node of key "{key}" from "{dataname}".\n\nParsed:\n{val}\n\nReal:\n{kwds[key]}' - assert type(kwds[key]) == type(val), f'check type of {i}-th node of key "{key}" from "{dataname}".\n\nParsed:\n{type(val)}\n\nReal:\n{type(kwds[key])}' # noqa: E721 + assert kwds[key] == val, ( + f'check value of {i}-th node of key "{key}" from "{dataname}".\n\nParsed:\n{val}\n\nReal:\n{kwds[key]}' + ) + assert type(kwds[key]) == type(val), ( # noqa: E721 + f'check type of {i}-th node of key "{key}" from "{dataname}".\n\nParsed:\n{type(val)}\n\nReal:\n{type(kwds[key])}' + ) - assert root.env.filename == oname + assert root.env.filename == str(oname) @pytest.mark.parametrize('dataname', get_datanames()) @@ -73,7 +77,6 @@ def test_picklable(dataname): pickle.dumps(root) - def test_iter_node(): root = loads(""" * H1 diff --git a/src/orgparse/tests/test_date.py b/src/orgparse/tests/test_date.py index 39de638..9764f97 100644 --- a/src/orgparse/tests/test_date.py +++ b/src/orgparse/tests/test_date.py @@ -10,7 +10,6 @@ def test_date_as_string() -> None: - testdate = datetime.date(2021, 9, 3) testdate2 = datetime.date(2021, 9, 5) testdatetime = datetime.datetime(2021, 9, 3, 16, 19, 13) diff --git a/src/orgparse/tests/test_hugedata.py b/src/orgparse/tests/test_hugedata.py index aaa7933..b97a178 100644 --- a/src/orgparse/tests/test_hugedata.py +++ b/src/orgparse/tests/test_hugedata.py @@ -8,16 +8,13 @@ def generate_org_lines(num_top_nodes, depth=3, nodes_per_level=1, _level=1): return for i in range(num_top_nodes): yield ("*" * _level) + f' {i}-th heading of level {_level}' - yield from generate_org_lines( - nodes_per_level, depth - 1, nodes_per_level, _level + 1) + yield from generate_org_lines(nodes_per_level, depth - 1, nodes_per_level, _level + 1) def num_generate_org_lines(num_top_nodes, depth=3, nodes_per_level=1): if depth == 0: return 0 - return num_top_nodes * ( - 1 + num_generate_org_lines( - nodes_per_level, depth - 1, nodes_per_level)) + return num_top_nodes * (1 + num_generate_org_lines(nodes_per_level, depth - 1, nodes_per_level)) def test_picklable() -> None: diff --git a/src/orgparse/tests/test_misc.py b/src/orgparse/tests/test_misc.py index 5c0b3ff..bb1382e 100644 --- a/src/orgparse/tests/test_misc.py +++ b/src/orgparse/tests/test_misc.py @@ -1,3 +1,7 @@ +import io + +import pytest + from orgparse.date import OrgDate from .. import load, loads @@ -17,12 +21,14 @@ def test_empty_heading() -> None: def test_root() -> None: - root = loads(''' + root = loads( + ''' #+STARTUP: hidestars Whatever # comment * heading 1 - '''.strip()) + '''.strip() + ) assert len(root.children) == 1 # todo not sure if should strip special comments?? assert root.body.endswith('Whatever\n# comment') @@ -82,8 +88,7 @@ def test_parse_custom_todo_keys(): env = OrgEnv(todos=todo_keys, dones=done_keys, filename=filename) root = loads(content, env=env) - assert root.env.all_todo_keys == ['TODO', 'CUSTOM1', - 'ANOTHER_KEYWORD', 'DONE', 'A'] + assert root.env.all_todo_keys == ['TODO', 'CUSTOM1', 'ANOTHER_KEYWORD', 'DONE', 'A'] assert len(root.children) == 5 assert root.children[0].todo == 'TODO' assert root.children[1].todo == 'DONE' @@ -107,28 +112,28 @@ def test_add_custom_todo_keys(): # after parsing, all keys are set root = loads(content, filename, env) - assert root.env.all_todo_keys == ['CUSTOM_TODO', 'COMMENT_TODO', - 'CUSTOM_DONE', 'COMMENT_DONE'] + assert root.env.all_todo_keys == ['CUSTOM_TODO', 'COMMENT_TODO', 'CUSTOM_DONE', 'COMMENT_DONE'] + def test_get_file_property() -> None: - content = """#+TITLE: Test: title + content = """#+TITLE: Test: title * Node 1 test 1 * Node 2 test 2 """ - # after parsing, all keys are set - root = loads(content) - assert root.get_file_property('Nosuchproperty') is None - assert root.get_file_property_list('TITLE') == ['Test: title'] - # also it's case insensitive - assert root.get_file_property('title') == 'Test: title' - assert root.get_file_property_list('Nosuchproperty') == [] + # after parsing, all keys are set + root = loads(content) + assert root.get_file_property('Nosuchproperty') is None + assert root.get_file_property_list('TITLE') == ['Test: title'] + # also it's case insensitive + assert root.get_file_property('title') == 'Test: title' + assert root.get_file_property_list('Nosuchproperty') == [] def test_get_file_property_multivalued() -> None: - content = """ #+TITLE: Test + content = """ #+TITLE: Test #+OTHER: Test title #+title: alternate title @@ -138,14 +143,13 @@ def test_get_file_property_multivalued() -> None: test 2 """ - # after parsing, all keys are set - root = loads(content) - import pytest + # after parsing, all keys are set + root = loads(content) - assert root.get_file_property_list('TITLE') == ['Test', 'alternate title'] - with pytest.raises(RuntimeError): - # raises because there are multiple of them - root.get_file_property('TITLE') + assert root.get_file_property_list('TITLE') == ['Test', 'alternate title'] + with pytest.raises(RuntimeError): + # raises because there are multiple of them + root.get_file_property('TITLE') def test_filetags_are_tags() -> None: @@ -163,7 +167,6 @@ def test_filetags_are_tags() -> None: def test_load_filelike() -> None: - import io stream = io.StringIO(''' * heading1 * heading 2 @@ -216,6 +219,7 @@ def test_level_0_timestamps() -> None: OrgDate((2019, 8, 10, 16, 30, 0), (2019, 8, 10, 17, 30, 0)), ] + def test_date_with_cookies() -> None: testcases = [ ('<2010-06-21 Mon +1y>', @@ -236,8 +240,8 @@ def test_date_with_cookies() -> None: "OrgDate((2019, 4, 5, 8, 0, 0), None, False, ('.+', 1, 'h'))"), ('<2007-05-16 Wed 12:30 +1w>', "OrgDate((2007, 5, 16, 12, 30, 0), None, True, ('+', 1, 'w'))"), - ] - for (inp, expected) in testcases: + ] # fmt: skip + for inp, expected in testcases: root = loads(inp) output = root[0].datelist[0] assert str(output) == inp @@ -247,8 +251,8 @@ def test_date_with_cookies() -> None: "OrgDate((2006, 11, 2, 20, 0, 0), (2006, 11, 2, 22, 0, 0), True, ('+', 1, 'w'))"), ('<2006-11-02 Thu 20:00--22:00 +1w>', "OrgDate((2006, 11, 2, 20, 0, 0), (2006, 11, 2, 22, 0, 0), True, ('+', 1, 'w'))"), - ] - for (inp, expected) in testcases: + ] # fmt: skip + for inp, expected in testcases: root = loads(inp) output = root[0].rangelist[0] assert str(output) == "<2006-11-02 Thu 20:00--22:00 +1w>" @@ -270,8 +274,8 @@ def test_date_with_cookies() -> None: ('* TODO Pay the rent\nDEADLINE: <2005-10-01 Sat .+1m>', "<2005-10-01 Sat .+1m>", "OrgDateDeadline((2005, 10, 1), None, True, ('.+', 1, 'm'))"), - ] - for (inp, expected_str, expected_repr) in testcases2: + ] # fmt: skip + for inp, expected_str, expected_repr in testcases2: root = loads(inp) output = root[1].deadline assert str(output) == expected_str @@ -292,8 +296,8 @@ def test_date_with_cookies() -> None: ('* TODO Pay the rent\nSCHEDULED: <2005-10-01 Sat .+1m>', "<2005-10-01 Sat .+1m>", "OrgDateScheduled((2005, 10, 1), None, True, ('.+', 1, 'm'))"), - ] - for (inp, expected_str, expected_repr) in testcases2: + ] # fmt: skip + for inp, expected_str, expected_repr in testcases2: root = loads(inp) output = root[1].scheduled assert str(output) == expected_str diff --git a/src/orgparse/tests/test_rich.py b/src/orgparse/tests/test_rich.py index e423b0d..5171bb0 100644 --- a/src/orgparse/tests/test_rich.py +++ b/src/orgparse/tests/test_rich.py @@ -1,6 +1,7 @@ ''' Tests for rich formatting: tables etc. ''' + import pytest from .. import loads @@ -36,7 +37,7 @@ def test_table() -> None: | value2 | ''') - [gap1, t1, gap2, t2, gap3, t3, gap4] = root.body_rich + [_gap1, t1, _gap2, t2, _gap3, t3, _gap4] = root.body_rich t1 = Table(root._lines[1:10]) t2 = Table(root._lines[11:19]) @@ -47,13 +48,12 @@ def test_table() -> None: assert ilen(t1.rows) == 6 with pytest.raises(RuntimeError): - list(t1.as_dicts) # not sure what should it be + list(t1.as_dicts) # not sure what should it be assert ilen(t2.blocks) == 2 assert ilen(t2.rows) == 5 assert list(t2.rows)[3] == ['[2020-11-05 Thu 23:44]', ''] - assert ilen(t3.blocks) == 2 assert list(t3.rows) == [['simple'], ['value1'], ['value2']] assert t3.as_dicts.columns == ['simple'] diff --git a/tox.ini b/tox.ini index c99ef94..a31cbab 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 3.21 # relies on the correct version of Python installed -envlist = ruff,tests,mypy +envlist = ruff,tests,mypy,ty # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333 # hack to prevent .tox from crapping to the project directory toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox @@ -9,7 +9,7 @@ toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox [testenv] # TODO how to get package name from setuptools? package_name = "orgparse" -passenv = +pass_env = # useful for tests to know they are running under ci CI CI_* @@ -18,6 +18,11 @@ passenv = MYPY_CACHE_DIR RUFF_CACHE_DIR +set_env = +# do not add current working directory to pythonpath +# generally this is more robust and safer, prevents weird issues later on + PYTHONSAFEPATH=1 + # default is 'editable', in which tox builds wheel first for some reason? not sure if makes much sense package = uv-editable @@ -26,7 +31,7 @@ package = uv-editable skip_install = true dependency_groups = testing commands = - {envpython} -m ruff check src/ \ + {envpython} -m ruff check \ {posargs} @@ -44,7 +49,19 @@ dependency_groups = testing commands = {envpython} -m mypy --no-install-types \ -p {[testenv]package_name} \ - # txt report is a bit more convenient to view on CI - --txt-report .coverage.mypy \ - --html-report .coverage.mypy \ + --txt-report .coverage.mypy \ + --html-report .coverage.mypy \ + # this is for github actions to upload to codecov.io + # sadly xml coverage crashes on windows... so we need to disable it + {env:CI_MYPY_COVERAGE} \ + {posargs} + + +[testenv:ty] +dependency_groups = testing +extras = optional +deps = # any other dependencies (if needed) +commands = + {envpython} -m ty \ + check \ {posargs} From c45bdd5052ad38b78d6df819c4e0620fa5bf41e5 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Thu, 30 Oct 2025 22:15:04 +0000 Subject: [PATCH 89/90] readme: add project status section see https://github.com/karlicoss/orgparse/issues/74 --- README.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.rst b/README.rst index 130e7ac..4020edf 100644 --- a/README.rst +++ b/README.rst @@ -96,3 +96,13 @@ True 'some text' >>> node.body ' Body texts...' + + +Project status +-------------- + +Project is maintained by @karlicoss (myself). + +For my personal use, orgparse mostly has all features I need, so there hasn't been much active development lately. + +However, contributions are always welcome! Please provide tests along with your contribution if you're fixing bugs or adding new functionality. From 9982e9aff91f587408486eddead2d31cfa3d5d7f Mon Sep 17 00:00:00 2001 From: kklein Date: Sat, 4 May 2024 17:21:10 +0200 Subject: [PATCH 90/90] Mention installation option via conda-forge. --- README.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.rst b/README.rst index 4020edf..e26fae6 100644 --- a/README.rst +++ b/README.rst @@ -6,12 +6,23 @@ * `Documentation (Read the Docs) `_ * `Repository (at GitHub) `_ * `PyPI `_ +* `conda-forge `_ Install ------- +You can install ``orgpase`` via PyPI + +.. code-block:: console + pip install orgparse +or via conda-forge + +.. code-block:: console + + conda install orgparse -c conda-forge + Usage -----