-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathlogseq-org-roam.el
1431 lines (1323 loc) · 63.6 KB
/
logseq-org-roam.el
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
;;; logseq-org-roam.el --- Logseq Org-roam converter -*- coding: utf-8; lexical-binding: t; -*-
;; Copyright (C) 2024, Sylvain Bougerel
;; Author: Sylvain Bougerel <[email protected]>
;; Maintainer: Sylvain Bougerel <[email protected]>
;; URL: https://github.com/sbougerel/logseq-org-roam/
;; Keywords: tools outlines
;; Version: 1.0.0
;; Package-Requires: ((org-roam "2.2.2") (emacs "27.2") (org "9.3"))
;; URL: https://github.com/sbougerel/logseq-org-roam
;; This file is NOT part of GNU Emacs.
;; This program is free software: you can redistribute it and/or modify it under
;; the terms of the GNU General Public License as published by the Free Software
;; Foundation, either version 3 of the License, or (at your option) any later
;; version.
;;
;; This program is distributed in the hope that it will be useful, but WITHOUT
;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
;; FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
;; details.
;;
;; You should have received a copy of the GNU General Public License along with
;; this program. If not, see <https://www.gnu.org/licenses/>.
;;; Commentary:
;;
;; [](http://www.gnu.org/licenses/gpl-3.0.html)
;; [](https://github.com/sbougerel/logseq-org-roam/actions)
;;
;; This package provide facilities to convert Logseq files to `org-roam' files
;; and author missing `org-roam' files when necessary. It should be used on the
;; entire `org-roam-directory' at once. By default, it uses the `org-roam'
;; cache to ensure that only the necessary files are parsed. Finally, it
;; provides clear descriptions of the changes it performs. Overall, it makes
;; Logseq and `org-roam' interoperable to some extent.
;;
;;; Usage
;;
;; Call `logseq-org-roam' to convert Logseq files to `org-roam' files. This
;; function is the single entry point to this package. Its behaviour can be
;; modified with univeral arguments. See `logseq-org-roam' for more
;; information.
;;
;; *You should keep a backup of your `org-roam-directory' before using this
;; *package.*
;;
;;; Installation
;;
;; With `use-package.el' and `straight.el', you can grab this package from
;; Github with:
;;
;; (use-package logseq-org-roam
;; :straight (:host github
;; :repo "sbougerel/logseq-org-roam"
;; :files ("*.el")))
;;
;;; Interoperability with Logseq
;;
;; This package assumes that the `org-roam-directory' shares its location with
;; the Logseq graph. The expected file structure should be similar to:
;;
;; .
;; └── org-roam-directory/
;; ├── assets/ ;; attachements
;; ├── journals/ ;; journals in Logseq, dailies in `org-roam'
;; ├── logseq/ ;; private to Logseq, ignored by `org-roam'
;; └── pages/ ;; Logseq pages, `org-roam' files
;;
;; Sample `org-roam' configuration for interoperability with Logseq:
;;
;; (setq org-roam-directory "/path/to/org-roam-directory")
;; (setq org-roam-dailies-directory "journals/")
;; (setq org-roam-file-exclude-regexp "logseq/.*$") ;; exclude Logseq files
;; (setq org-roam-capture-templates
;; '(("d" "default" plain "%?"
;; ;; Accomodates for the fact that Logseq uses the "pages" directory
;; :target (file+head "pages/${slug}.org" "#+title: ${title}\n")
;; :unnarrowed t))
;; org-roam-dailies-capture-templates
;; '(("d" "default" entry "* %?"
;; :target (file+head "%<%Y-%m-%d>.org" "#+title: %<%Y-%m-%d>\n"))))
;;
;; Corresponding Logseq configuration for interoperability with `org-roam':
;;
;; :preferred-format :org ;; required
;; :pages-directory "pages" ;; default
;; :journals-directory "journals" ;; default, must match `org-roam-dailies-directory'
;; :journal/page-title-format "yyyy-MM-dd" ;; match with `org-roam-dailies-capture-templates'
;; :journal/file-name-format "yyyy-MM-dd" ;; match with `org-roam-dailies-capture-templates'
;; :preferred-workflow :todo ;; recommended
;; :property-pages/enabled? false ;; recommended, disable property pages
;;
;; Finally, provide the following settings for `logseq-org-roam':
;;
;; (setq logseq-org-roam-link-types 'fuzzy) ;; or 'files, depending on the
;; ;; setting ":org-mode/insert-file-link?"
;; ;; See `logseq-org-roam-link-types'
;; (setq logseq-org-roam-pages-directory "pages")
;; (setq logseq-org-roam-journals-directory "journals")
;; (setq logseq-org-roam-journals-file-name-format "%Y-%m-%d")
;; (setq logseq-org-roam-journals-title-format "%Y-%m-%d")
;;
;; With the configurations above, this package makes it possible to use Logseq
;; and `org-roam' together.
;;
;;; Caveats
;;
;; This package assumes that you prefer to use `org-roam' as your primary note
;; taking tool; and Logseq as a companion. When it converts a file from Logseq
;; to `org-roam', it can update it link to use ID links. Logseq recognizes ID
;; links for navigation, but does not reference them properly as of version
;; 0.10.3: this means that once converted to an ID link, the backlink will show
;; under "Unlinked references" in Logseq. This limitation extends to the graph;
;; the link will not be shown in the graph. In fact, a fully converted set of
;; notes (where all links are ID links) will result in an empty graph. However
;; all files are still searchable and navigable in Logseq. If you care about
;; the graph in Logseq, this package is not for you.
;;
;; Logseq's workflow often results in leftovers "*" at the bottom files. In
;; some versions of `org', the `org-element' parser may throw errors
;; (e.g. "wrong-side-of-point"). These errors prevent `org-roam' from parsing
;; the buffer properly and updating the database. Try to keep your files tidy
;; in Logseq and not leave trailing stars.
;;
;; This package only deal with updating a file's first section (top properties,
;; title, aliases) and its links. Converting timestamps, tags, asset location
;; is not supported. In some cases, interoperability can be maintained by
;; keeping some discipline or aligning configurations (e.g. for templates).
;;
;; Finally, the changes done by the package are destructive, and backup copies
;; are not kept by this package. The author uses version control.
;;
;;; History
;;
;; This package is based on an original idea by William R Burdick Jr
;; (https://gist.github.com/zot/ddf1a89a567fea73bc3c8a209d48f527) and its port
;; to a package by Ivan Danov (https://github.com/idanov/org-roam-logseq.el).
;; It is a complete rewrite.
;;
;; Some limitations in the orignal works above motivated me to write a new
;; package:
;;
;; - `logseq-org-roam' is the single entry point
;;
;; - `logseq-org-roam' supports multiple `org-roam' directories (Logseq graphs)
;;
;; - `logseq-org-roam' has a more robust logic to match Logseq links to
;; `org-roam' nodes. For example, it does not confuse internal links, it
;; supports aliases, disregards case-sensitivity for the file's base name, and
;; more.
;;
;; - `logseq-org-roam' provides better support for journal dates.
;;
;; - `logseq-org-roam' is able to add new entries from new (dead) links created
;; in Logseq, as you normally would in the `org-roam' workflow, since Logseq
;; does not do it. Which in turn makes `org-roam' cache usage even more
;; relevant.
;;
;; - `logseq-org-roam' is verbose about the changes it makes.
;;
;; - Finally, `logseq-org-roam' should be rather fast; it should be able to
;; handle 500 files in about 10 seconds (when ignoring the cache)
;;
;; It's not all better. `logseq-org-roam' has some drawbacks compared to the
;; original works above:
;;
;; - No single-file update is provided, as in `org-roam-logseq.el'.
;;
;; - `logseq-org-roam' is considerably more complex (more than the author
;; anticipated). Since it is a larger package, it should pay off to lazy-load
;; it. To this end, autoload cookies are assigned to `logseq-org-roam' as
;; well as customization options.
;;; Code:
(require 'org)
(require 'org-roam)
(require 'image) ;; for `image-type-file-name-regexps' & `image-type-available-p'
(defgroup logseq-org-roam nil
"Convert Logseq files to `org-roam' files."
:group 'org-roam)
;;;###autoload (put 'logseq-org-roam-link-types 'safe-local-variable #'symbolp)
(defcustom logseq-org-roam-link-types nil
"The kind of links `logseq-org-roam' should convert.
Value is a symbol, only the following are recognized:
- \\='files
- \\='fuzzy
- nil (default, if unrecognized)
You should customize this value based on your
\":org-mode/insert-file-link?\" setting in Logseq. Values other
than nil save some processing time.
Links considered as candidates to be converted to `org-roam'
ID links are of 2 types:
- File links such as:
[[file:path/to/pages/page.org][DESCRIPTION]].
- Fuzzy links such as [[TITLE-OR-ALIAS][DESCRIPTION]].
Matching rules for each kind of links are as follows.
When dealing with file links, `logseq-org-roam' ignores links
that do not contain a description since Logseq always populates
it when referencing another page. It also ignores links that
contain search options since Logseq does not create those. And
finally it discards any links that is not a link to an `org-roam'
file (since these are not convertible to ID links).
When dealing with fuzzy links, it first ignores dedicated internal
link formats that have specific meaning in `org-mode' (even if
they are broken):
- [[#custom-id]] links,
- [[*heading]] links,
- [[(coderef)]] links,
- [[image.jpg]] inline links to images,
Of the remaining fuzzy links, it discards links that match
internally (as per `org-mode' rules) with:
- <<targets>> or,
- #+name: named elements or,
- a headline by text search,
The leftover links are the candidates to be converted to
`org-roam' external ID links.
Notes on using file links in Logseq.
It is usually recommended to set \":org-mode/insert-file-link?\"
to true in Logseq, presumably to ensure the correct target is
being pointed to.
Unfortunately, Logseq does not always provide a correct path (as
of version 0.10.3) on platforms tested (Android, Linux). As of
version 0.10.3, when a note does not exist yet (or when it is
aliased, see `https://github.com/logseq/logseq/issues/9342'), the
path provided by Logseq is incorrect. (TODO: test in newer
versions.)
On the other hand `logseq-org-roam' cares to implement the
complex matching rules set by `org-roam' to convert the right
fuzzy links, making Logseq and `org-roam' mostly interoperable
even when using fuzzy links in Logseq."
:type 'symbol
:options '(fuzzy file both)
:group 'logseq-org-roam)
;;;###autoload (put 'logseq-org-roam-pages-directory 'safe-local-variable #'string)
(defcustom logseq-org-roam-pages-directory "pages"
"Set this variable to mirror Logseq :pages-directory setting."
:type 'string
:group 'logseq-org-roam)
;;;###autoload (put 'logseq-org-roam-journals-directory 'safe-local-variable #'string)
(defcustom logseq-org-roam-journals-directory "journals"
"Set this variable to mirror Logseq :journals-directory setting."
:type 'string
:group 'logseq-org-roam)
;;;###autoload (put 'logseq-org-roam-journals-file-name-format 'safe-local-variable #'string)
(defcustom logseq-org-roam-journals-file-name-format "%Y-%m-%d"
"Set this variable to mirror Logseq :journal/file-name-format setting.
You should pick a format that `logseq-org-roam-maybe-date-func'
can use. Otherwise, titles for journal entries will not be
formated correctly: `logseq-org-roam' first parses the file name
into a time before feeding it back to `format-time-string' to
create the title (See: `logseq-org-roam-jounals-title-format')."
:type 'string
:group 'logseq-org-roam)
;;;###autoload (put 'logseq-org-roam-journals-title-format 'safe-local-variable #'string)
(defcustom logseq-org-roam-journals-title-format "%Y-%m-%d"
"Set this variable to mirror Logseq :journal/file-name-format setting.
This is used to create a title for journal entires and to find
out which fuzzy links point to journal entries (See
`logseq-org-roam-maybe-date-func').
You can set this to any format that `format-time-string' accepts.
However, you should only use it to create date strings, and not
time strings. Having hours and seconds in the format will
make it impossible to find out journal entries from fuzzy links."
:type 'string
:group 'logseq-org-roam)
;;;###autoload (put 'logseq-org-roam-maybe-date-func 'safe-local-variable #'symbolp)
(defcustom logseq-org-roam-maybe-date-func
#'logseq-org-roam-maybe-date-default
"Try parsing a string into a date and return time when successful.
When non-nil, this variable is called with `funcall'. It is
given 2 arguments: the first is a time format for
`format-time-string', the second is the string to evaluate. It
is expected to return a time, like `date-to-time' or
`encode-time'. If the time returned is 0, it assumes that the
string is not a date. See `logseq-org-roam-maybe-date-default'
for a description of the default behaviour.
If nil, date parsing is disabled."
:type 'string
:group 'logseq-org-roam)
(defcustom logseq-org-roam-create-replace '(("[\\/]" . "_"))
"Alist specifying replacements for fuzzy links.
Car and cdr of each cons will be given as arguments to
`replace-regexp-in-string' when converting fuzzy links to paths
in `logseq-org-roam-create-translate-default'."
:type 'alist
:group 'logseq-org-roam)
;;;###autoload (put 'logseq-org-roam-create-translate-func 'safe-local-variable #'symbolp)
(defcustom logseq-org-roam-create-translate-func
#'logseq-org-roam-create-translate-default
"Function translating a fuzzy link to a file path.
When non-nil, it is called with `funcall' and a single argument,
the fuzzy link. It is expected to return an absolute file path.
This variable provide complete control over how fuzzy links are
translated to file paths.
Default to `logseq-org-roam-create-translate-default'. Setting
this value to nil disables creation of pages for fuzzy links."
:type 'symbol
:group 'logseq-org-roam)
;;;###autoload (put 'logseq-org-roam-create-accept-func 'safe-local-variable #'symbol)
(defcustom logseq-org-roam-create-accept-func #'logseq-org-roam-pages-p
"Tells aparts paths that should be created from paths that should not.
When non-nil, it is called as a function with a single argument:
the path. When the return value is non-nil, the path is accepted
and the file is created.
The default value (`logseq-org-roam-pages-p') will not create
journal entires. If you want journal entries to be created too,
you can set this to `logseq-org-roam-logseq-p'. If you want to
allow files to be created anywhere, you can set this to `always'.
When set to nil, file creation is disabled."
:type 'symbol
:group 'logseq-org-roam)
;;;###autoload
(defcustom logseq-org-roam-updated-hook nil
"Hook called by `logseq-org-roam' if any files was updated."
:type 'hook
:group 'logseq-org-roam)
(defconst logseq-org-roam--named
'(babel-call
center-block
dynamic-block
example-block
export-block
fixed-width
footnote-definition
horizontal-rule
latex-environment
paragraph
plain-list
quote-block
special-block
src-block
table
verse-block)
"List of org-elements that can be affiliated with a :name attribute.")
(defconst logseq-org-roam--log-buffer-name "*Logseq Org-roam %s*"
"Name for the log buffer.
'%s' will be replaced by `org-roam-directory' if present")
(defmacro logseq-org-roam--with-log-buffer (&rest body)
"Bind standard output to a dedicated buffer for the duration of BODY."
(declare (debug t))
`(let* ((standard-output
(with-current-buffer
(get-buffer-create
;; One buffer per org-roam-directory
(format logseq-org-roam--log-buffer-name
org-roam-directory))
(kill-all-local-variables) ;; return to fundamental for logging
(setq default-directory org-roam-directory)
(setq buffer-read-only nil)
(setq buffer-file-name nil)
(setq buffer-undo-list t)
(goto-char (point-max))
(if (/= (point-min) (point-max))
(insert "\n\n"))
(current-buffer))))
(prog1 (progn ,@body)
(with-current-buffer standard-output
(goto-char (point-max))
(insert "You can set this buffer to `org-mode' to navigate links\n")
(setq buffer-read-only t)))))
(defmacro logseq-org-roam--with-edit-buffer (file &rest body)
"Find an existing buffer for FILE, set `org-mode' and execute BODY.
If the buffer is new, `org-mode' startup is inhibited. This
macro does not save the file, but will *always* kill the buffer
if it was previously created."
(declare (indent 1) (debug t))
(let ((exist-buf (make-symbol "exist-buf"))
(buf (make-symbol "buf"))
(bimf (make-symbol "bimf"))
(biro (make-symbol "biro")))
`(let* ((,bimf inhibit-modification-hooks)
(,biro inhibit-read-only)
(,exist-buf (find-buffer-visiting ,file))
(,buf
(or ,exist-buf
(let ((auto-mode-alist nil)
(find-file-hook nil))
(find-file-noselect ,file)))))
(unwind-protect
(with-current-buffer ,buf
(setq inhibit-read-only t)
(setq inhibit-modification-hooks (if ,exist-buf t ,bimf))
(unless (derived-mode-p 'org-mode)
(let ((org-inhibit-startup t)) (org-mode)))
,@body)
(setq inhibit-modification-hooks ,bimf)
(setq inhibit-read-only ,biro)
(unless ,exist-buf (kill-buffer ,buf))))))
(defmacro logseq-org-roam--with-temp-buffer (file &rest body)
"Visit FILE into an `org-mode' temp buffer and execute BODY.
If the buffer is new, `org-mode' startup is inhibited. This
macro does not save the file, but will kill the buffer if it was
previously created."
(declare (indent 1) (debug t))
`(with-temp-buffer
;; relative path expansion needs this
(setq default-directory (file-name-directory ,file))
(delay-mode-hooks
(let ((org-inhibit-startup t)) (org-mode)))
(insert-file-contents ,file)
,@body))
(defmacro logseq-org-roam--catch-fun (sym errs fun &rest body)
"Catch ERRS for SYM during BODY's execution and pass it to FUN."
(declare (indent 3) (debug t))
(let ((result (make-symbol "result")))
`(let ((,result (catch ,sym ,@body)))
(if (and ,result
(symbolp ,result)
(memq ,result ,errs))
(apply ,fun (list ,result))))))
(defun logseq-org-roam--fl (file)
"Make an org link to FILE relative to `org-roam-directory'."
(format "[[file:%s][%s]]"
file
(file-relative-name file org-roam-directory)))
(defun logseq-org-roam-pages-p (file)
"Return non-nil if FILE path is under the Logseq pages directory."
(string= (directory-file-name (file-name-directory file))
(expand-file-name logseq-org-roam-pages-directory
org-roam-directory)))
(defun logseq-org-roam-journals-p (file)
"Return non-nil if FILE path is under the Logseq journal directory."
(string= (directory-file-name (file-name-directory file))
(expand-file-name logseq-org-roam-journals-directory
org-roam-directory)))
(defun logseq-org-roam-logseq-p (file)
"Return non-nil if FILE path is under the Logseq journal or pages directory."
(or (logseq-org-roam-pages-p file)
(logseq-org-roam-journals-p file)))
(defun logseq-org-roam--image-file-p (file)
"Non-nil if FILE is a supported image type."
;; This function exists purely because `image-supported-file-p' has made
;; `image-type-from-file-name' obsolete since Emacs 29.1; while it is not
;; supported by `compat'. So `image-type-available-p' is just copied here
;;
;; The function below is a copy from `image.el' distributed with Emacs version
;; 29.1. Copyright (C) 1998-2023 Free Software Foundation, Inc.
(let ((case-fold-search t)
type)
(catch 'found
(dolist (elem image-type-file-name-regexps)
(if (and (string-match-p (car elem) file)
(image-type-available-p (setq type (cdr elem))))
(throw 'found type))))))
(defun logseq-org-roam--value-string-p (element)
"Return non-nil if ELEMENT has a string value that is not empty."
(and (org-element-property :value element)
(not (string-empty-p (org-element-property :value element)))))
(defun logseq-org-roam-maybe-date-default (date-format maybe-date)
"When MAYBE-DATE match DATE-FORMAT, turn it into a time value.
Attempts to parse MAYBE-DATE with `parse-time-string' first and
convert it back to a string with `format-time-string' using
DATE-FORMAT. If both string match, it is taken as a journal
date, and the corresponding time is returned."
;; TODO: Compile a reverse regex to format-time-string?
;; NOTE: hack because '_' are not ISO date chars
(let* ((hacked-date (replace-regexp-in-string "_" "-" maybe-date))
(parsed (parse-time-string hacked-date))
(year (nth 5 parsed))
(month (nth 4 parsed))
(day (nth 3 parsed))
(time (encode-time `(0 0 0 ,day ,month ,year nil -1 nil))))
(if (and year month day
(string= (format-time-string date-format time)
maybe-date))
time
0)))
(defun logseq-org-roam--inventory-init (files)
"Initialise inventory with FILES."
(let ((inventory (make-hash-table :test #'equal)))
(mapc (lambda (elem) (puthash elem nil inventory)) files)
inventory))
(defun logseq-org-roam--inventory-from-cache (inventory)
"Populate INVENTORY with `org-roam' cache for FILES.
Return the number of files whose metadata was retreived from the
cache."
(let ((count 0)
(data-cached (org-roam-db-query [:select [file id title]
:from nodes
:where (= 0 level)])))
(pcase-dolist (`(,file ,id ,title) data-cached)
;; TODO: Consider throwing an error if cache is not updated
(unless (eq 'not-found (gethash file inventory 'not-found))
(setq count (1+ count))
(let ((aliases (mapcar #'car
(org-roam-db-query [:select [alias] :from aliases
:where (= node-id $s1)]
id))))
(puthash file
(append (list :cache-p t :id id)
(if (and title (not (string-empty-p title)))
(list :title title))
(if aliases
(list :roam-aliases aliases)))
inventory))))
count))
(defun logseq-org-roam--inventory-mark-external (files inventory)
"Mark FILES in INVENTORY that are not Logseq files."
(let ((count 0))
(mapc (lambda (file)
(unless (logseq-org-roam-logseq-p file)
(setq count (1+ count))
(let* ((plist (gethash file inventory))
(new_plist (plist-put plist :external-p t)))
(puthash file new_plist inventory))))
files)
count))
(defun logseq-org-roam--inventory-mark-modified (files inventory)
"Update INVENTORY with modified buffer visiting any FILES.
Return the number of files from INVENTORY that are currently
being modified in a buffer."
(let ((count 0))
(dolist (file files)
(when-let* ((existing_buf (find-buffer-visiting file))
(mod-p (buffer-modified-p existing_buf)))
(setq count (1+ count))
(let* ((plist (gethash file inventory))
(new_plist (plist-put plist :modified-p t)))
(puthash file new_plist inventory))))
count))
(defun logseq-org-roam--parse-first-section-properties (section plist)
"Return updated PLIST based on first SECTION properties."
(org-element-map section 'property-drawer
(lambda (property-drawer)
;; Set `:title-point' after the drawer, and reset if there's a title
(setq plist (plist-put plist :title-point
(progn
(goto-char
(- (org-element-property :end
property-drawer)
(org-element-property :post-blank
property-drawer)))
(beginning-of-line)
(point))))
(org-element-map property-drawer 'node-property
(lambda (node-property)
(let ((key (org-element-property :key node-property)))
(cond
((and (string= "ID" key)
(logseq-org-roam--value-string-p node-property))
(setq plist (plist-put plist :id
(org-element-property :value
node-property))))
((and (string= "ROAM_ALIASES" key)
(logseq-org-roam--value-string-p node-property))
(setq plist (plist-put plist :roam-aliases
(split-string-and-unquote
(org-element-property
:value
node-property)))))))))))
plist)
(defun logseq-org-roam--parse-first-section-keywords (section plist)
"Return updated PLIST based on first SECTION keywords."
(org-element-map section 'keyword
(lambda (keyword)
(let ((key (org-element-property :key keyword)))
(cond
((string= "TITLE" key)
(setq plist (plist-put plist :title-point
(org-element-property :begin keyword)))
(if (logseq-org-roam--value-string-p keyword)
(setq plist (plist-put plist :title
(org-element-property :value keyword)))))
((and (string= "ALIAS" key)
(logseq-org-roam--value-string-p keyword))
(setq plist (plist-put plist :aliases
(split-string
(org-element-property :value keyword)
"\\s-*,\\s-*"))))))))
plist)
(defun logseq-org-roam--parse-first-section (data plist)
"Return updated PLIST based on first section of DATA."
(declare (pure t) (side-effect-free t))
(cond
((or (not (consp data))
(not (eq 'org-data (car data))))
(throw 'parse-error 'invalid-ast))
((or (not (cddr data))
(not (consp (caddr data)))
(not (eq 'section (caaddr data))))
;; no content or no first section
plist)
(t
(setq plist (plist-put plist :first-section-p t))
(let ((section (caddr data)))
(setq plist (logseq-org-roam--parse-first-section-properties section plist))
(setq plist (logseq-org-roam--parse-first-section-keywords section plist)))
plist)))
(defun logseq-org-roam--parse-file-links (data plist)
"Return updated PLIST based on file links in DATA."
(let ((links (plist-get plist :links)))
(org-element-map data 'link
(lambda (link)
(if (and (string= (org-element-property :type link) "file")
(org-element-property :contents-begin link)
(not (org-element-property :search-option link)))
(let* ((path (org-element-property :path link))
(descr (buffer-substring-no-properties
(org-element-property :contents-begin link)
(org-element-property :contents-end link)))
(begin (org-element-property :begin link))
(end (- (org-element-property :end link)
(org-element-property :post-blank link)))
(raw (buffer-substring-no-properties begin end)))
(push (list 'file begin end path descr raw) links)))))
(if links (setq plist (plist-put plist :links links))))
plist)
(defun logseq-org-roam--parse-fuzzy-links (data plist)
"Return updated PLIST based on fuzzy links in DATA.
This function ensures that we do not convert fuzzy links that
already match internal links; they take precedence over external
ID links."
(let ((links (plist-get plist :links))
(text-targets (make-hash-table :test #'equal)))
(org-element-map data (append '(link target headline)
logseq-org-roam--named)
(lambda (element)
(let ((type (org-element-type element)))
(if-let ((name (org-element-property :name element)))
;; Org-mode searches are case incensitive
(puthash (downcase name) t text-targets))
(cond
;; See `org-link-search' to understand what fuzzy link point to
((eq type 'headline)
(puthash (downcase
;; TODO: remove internal function dependency
;; This function skips tasks, priority, a statistics cookies
(org-link--normalize-string
(org-element-property :raw-value element))) t text-targets))
((and (eq type 'link)
(string= (org-element-property :type element) "fuzzy"))
(let ((path (org-element-property :path element))
(content (org-element-property :contents-begin element)))
(unless (or
;; "[[*Heading]]" links qualify as internal, ignore them
(string-match "\\`\\*" path)
;; When the link has no content, ignore image types
(and (not content)
(logseq-org-roam--image-file-p path)))
(let* ((descr (if content
(buffer-substring-no-properties
(org-element-property :contents-begin element)
(org-element-property :contents-end element))))
(begin (org-element-property :begin element))
(end (- (org-element-property :end element)
(org-element-property :post-blank element)))
(raw (buffer-substring-no-properties begin end)))
(push (list 'fuzzy begin end path descr raw) links)))))
((eq type 'target)
(puthash (downcase (org-element-property :value element)) t text-targets))))))
;; Filter out link that match targets, headlines or named elements
(setq links (seq-filter (lambda (link) (not (gethash (nth 3 link) text-targets)))
links))
(if links (setq plist (plist-put plist :links links))))
plist)
(defun logseq-org-roam--parse-buffer (plist parts)
"Return updated PLIST based on current buffer's content.
This function updates PLIST with based on selected PARTS, a list
of keywords which defaults to \\='(first-section file-links
fuzzy-links)."
(org-with-wide-buffer
(let* ((data (org-element-parse-buffer)))
(if (memq 'first-section parts)
(setq plist (logseq-org-roam--parse-first-section data plist)))
;; links are never updated for external files
(unless (plist-get plist :external-p)
(if (memq 'file-links parts)
(setq plist (logseq-org-roam--parse-file-links data plist)))
(if (memq 'fuzzy-links parts)
(setq plist (logseq-org-roam--parse-fuzzy-links data plist))))))
plist)
(defun logseq-org-roam--parse-files (files inventory parts)
"Populate INVENTORY by parsing content of FILES.
Restrict parsing to PARTS if provided. Return the number of files
that were parsed."
(let ((count 0))
(dolist (file files)
;; It's OK if plist is nil!
(let ((plist (gethash file inventory)))
(unless (or (plist-get plist :modified-p)
(plist-get plist :cache-p)
(plist-get plist :parse-error)
(plist-get plist :update-error))
(logseq-org-roam--catch-fun
'parse-error '(invalid-ast)
(lambda (err)
(princ (concat "- Error parsing " (logseq-org-roam--fl file) "\n"))
(puthash file
(plist-put plist :parse-error err)
inventory))
(logseq-org-roam--with-temp-buffer file
(let ((new_plist (plist-put plist :hash
(secure-hash
'sha256 (current-buffer)))))
(setq new_plist
(logseq-org-roam--parse-buffer new_plist parts))
(puthash file new_plist inventory)
(princ (concat "- Parsed " (logseq-org-roam--fl file) "\n"))
(setq count (1+ count))))))))
count))
(defun logseq-org-roam--inventory-all (files inventory force parts)
"Build inventory of `org-roam' metadata for FILES.
Update INVENTORY (hashtable) with a plist describing relevant
metadata to convert Logseq files to `org-roam'. The
keys (absolute paths) point to both existing and new `org-roam'
files (presumably created with Logseq).
The argument FORCE ensure that all files are parsed, instead of
relying on information from the `org-roam' cache (in which case,
files already indexed are ever modififed).
The argument PARTS ensures that the function only parses the
necessary parts of each files.
Returns the number of files parsed without error."
(princ "** Initial inventory:\n")
(let* (count_cached
count_external
count_modified
count_parsed)
(unless force
(setq count_cached
(logseq-org-roam--inventory-from-cache inventory)))
(setq count_modified
(logseq-org-roam--inventory-mark-modified files inventory))
(setq count_external
(logseq-org-roam--inventory-mark-external files inventory))
(setq count_parsed
(logseq-org-roam--parse-files files inventory parts))
(princ
(concat
(format "%s files found in org-roam directory\n"
(hash-table-count inventory))
(unless force
(format "%s files' metadata retrieved from cache\n"
count_cached))
(format "%s files being visited in a modified buffer will be skipped\n"
count_modified)
(format "%s files are external to Logseq and will not be modified\n"
count_external)
(format "%s files have been parsed without errors\n"
count_parsed)))
count_parsed))
(defun logseq-org-roam--inventory-update (files inventory parts)
"Update INVENTORY by reparsing FILES."
(princ "** Inventory update:\n")
(let* (count_parsed)
(setq count_parsed
(logseq-org-roam--parse-files files inventory parts))
(princ
(concat
(format "%s files have been parsed without errors\n"
count_parsed)))))
(defalias 'logseq-org-roam--normalize-text (symbol-function #'downcase)
"Return a normalized version of TEXT.")
;; Maintain optimizations
(put 'logseq-org-roam--normalize-text 'side-effect-free t)
(put 'logseq-org-roam--normalize-text 'byte-compile 'byte-compile-one-arg)
(put 'logseq-org-roam--normalize-text 'byte-opcode 'byte-downcase)
(defun logseq-org-roam--fill-fuzzy-dict (fuzzy-dict files inventory)
"Fill titles and aliases of FILES into FUZZY-DICT.
Map each title and alias to a key in INVENTORY or to a `cons'
containing a key to INVENTORY when a conflict is found.
Ensure that titles and aliases found across all files are unique.
If any 2 titles or alias conflicts with each other, there is no
unique target for titled links to these files.
Returns the number of conflicts found"
(princ "** Filling dictionary of titles and aliases:\n")
(let ((conflict-count 0))
(dolist (file files)
(when-let ((plist (gethash file inventory)))
(unless (or (plist-get plist :modified-p)
(plist-get plist :parse-error)
(plist-get plist :update-error))
;; NOTE: Similar title and aliases from the same file are not marked as conflict
;; TODO: cl-* could be faster
(let ((merged (seq-uniq
(mapcar #'logseq-org-roam--normalize-text
(delq nil (append
(list (plist-get plist :title))
(plist-get plist :aliases)
(plist-get plist :roam-aliases))))
#'string=)))
(dolist (target merged)
(let ((val (gethash target fuzzy-dict 'not-found))
other-file)
(if (eq 'not-found val)
(puthash target file fuzzy-dict)
(setq conflict-count (1+ conflict-count))
(if (consp val)
(setq other-file (car val))
(setq other-file val)
(puthash target (cons other-file nil) fuzzy-dict))
;; Log the conflict
(let* ((this-title-p (string= target (logseq-org-roam--normalize-text
(plist-get plist :title))))
(other-plist (gethash other-file inventory))
(other-title-p (string= target (logseq-org-roam--normalize-text
(plist-get other-plist :title)))))
(princ (format "- The %s \"%s\" in %s conflicts with the %s in %s, links will not be converted.\n"
(if this-title-p "title" "alias")
target
(logseq-org-roam--fl file)
(if other-title-p "title" "alias")
(logseq-org-roam--fl other-file)))))))))))
(princ (format "%s entries in total\n" (hash-table-count fuzzy-dict)))
(princ (format "%s conflicts\n" conflict-count))
conflict-count))
(defun logseq-org-roam--normalize-path (path)
"Return a new PATH that is normalized.
The file base portion of path will be downcased, and lowbar
repetitions will be removed. This helps with mapping
\"triple-lowbar\" setting in Logseq to file slug created by
`org-roam'."
(let ((dir (file-name-directory path))
(ext (file-name-extension path))
(base (file-name-base path)))
(concat dir
(replace-regexp-in-string "_+" "_" (downcase base))
"." ext)))
(defun logseq-org-roam--fill-file-dict (file-dict files)
"Fill FILE-DICT with the normalized path of each FILES.
Maps each normalized path to the original path or to a `cons' with
original path. If the mapping is to a cons, it means a conflict
was found and the `car' contains the first path to this
entry."
(princ "** Filling dictionary of similar paths:\n")
(let ((conflict-count 0))
(dolist (file files)
(let ((normalized (logseq-org-roam--normalize-path file)))
(if (eq 'not-found (gethash normalized file-dict 'not-found))
(puthash normalized file file-dict)
(setq conflict-count (1+ conflict-count))
(let ((val (gethash normalized file-dict))
original)
(if (consp val)
(setq original (car val))
(puthash normalized (cons val nil) file-dict)
(setq original val))
(princ (format "- Path to %s and %s are too similar and will not be converted\n"
(logseq-org-roam--fl file)
(logseq-org-roam--fl original)))))))
(princ (format "%s entries in total\n" (hash-table-count file-dict)))
(princ (format "%s conflicts\n" conflict-count))
conflict-count))
(defun logseq-org-roam--buffer-title ()
"Return a title based on current buffer's file name.
If the file is a journal entry, format the title accroding to
`logseq-org-roam-journals-title-format'."
(let* ((base-name (file-name-base (buffer-file-name))))
(if (logseq-org-roam-journals-p (buffer-file-name))
(let ((time (condition-case ()
(funcall logseq-org-roam-maybe-date-func
logseq-org-roam-journals-file-name-format
base-name)
(error 0))))
(if (time-equal-p 0 time) base-name
(format-time-string logseq-org-roam-journals-title-format time)))
base-name)))
(defun logseq-org-roam--update-first-section (plist)
"Update current buffer first section based on PLIST."
(org-with-wide-buffer
(let ((start-size (buffer-size))
(first-section-p (plist-get plist :first-section-p))
(beg (point-min)))
(goto-char beg)
(unless first-section-p
(insert "\n") ;; Empty line is needed to create first section
(backward-char))
(unless (plist-get plist :id)
(org-id-get-create))
(unless (plist-get plist :title)
(let ((title (logseq-org-roam--buffer-title)))
(goto-char (+ (or (plist-get plist :title-point) beg)
(- (buffer-size) start-size)
(if first-section-p 0 -1)))
(unless (bolp) (throw 'update-error 'mismatch-before-title))
(insert (concat "#+title: " title "\n"))))
(when-let ((diff (seq-difference (plist-get plist :aliases)
(plist-get plist :roam-aliases))))
(goto-char beg)
(dolist (alias diff)
(org-roam-property-add "ROAM_ALIASES" alias)))
(unless first-section-p
(goto-char (+ beg (- (buffer-size) start-size) -1))
(unless (looking-at "\n") (throw 'update-error 'mismatch-first-section))
(delete-char 1)))))
(defun logseq-org-roam--update-links (links inventory file-dict fuzzy-dict)
"Convert LINKS in current buffer to a target in INVENTORY.
This function returns t or an error code if there was an issue
updating the buffer.
The argument FILE-DICT is a hash-table that maps a normalized
file path to a key in inventory (a file path). When dealing only
with fuzzy links, this hashtable is not used.
The argument FUZZY-DICT is a hash-table that maps a normalized
fuzzy link to a key in inventory (a file path). When dealing
only with file links, this hashtable is not used."
;; `secure-hash' has a small chance of collision
(pcase-dolist (`(_ ,beg ,end _ _ ,raw) links)
(unless (string= (buffer-substring-no-properties beg end) raw)
(throw 'update-error 'mismatch-link)))
(pcase-dolist (`(,type ,beg ,end ,path ,descr _)
;; Avoid offset calculations with buffer updates
(sort links (lambda (a b) (> (nth 1 a) (nth 1 b)))))
(when-let ((id (plist-get
(gethash
;; file-dict and fuzzy-dict key can be `consp' (conflict)
(if (eq 'file type)
(gethash (logseq-org-roam--normalize-path
(expand-file-name path)) file-dict)
(gethash (logseq-org-roam--normalize-text path) fuzzy-dict))
inventory) :id)))
(save-excursion
(save-restriction
(narrow-to-region beg end)
(goto-char beg)
(delete-region beg end)
;; TODO: log link updates
(if descr
(insert (concat "[[id:" id "][" descr "]]"))
(insert (concat "[[id:" id "][" path "]]"))))))))
(defun logseq-org-roam--update-all (files inventory &optional link-p file-dict fuzzy-dict)
"Update all FILES according to INVENTORY.
By default only the first section is updated, but if LINK-P is
non-nil, links are updated instead taking into account
FILE-DICT and FUZZY-DICT.