From b06e23064d042fff848fe65a89127c52e26911ec Mon Sep 17 00:00:00 2001
From: Philippa Markovics <philippa@markovics.com>
Date: Tue, 27 Jun 2023 15:35:09 +0200
Subject: [PATCH 1/5] Mixed Clojure and Markdown parser WIP

---
 demo/src/nextjournal/clojure_mode/demo.cljs | 27 ++++++++++++++++++++-
 src/nextjournal/clojure_mode.cljs           |  6 ++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/demo/src/nextjournal/clojure_mode/demo.cljs b/demo/src/nextjournal/clojure_mode/demo.cljs
index bc98462f..ff7024cd 100644
--- a/demo/src/nextjournal/clojure_mode/demo.cljs
+++ b/demo/src/nextjournal/clojure_mode/demo.cljs
@@ -1,8 +1,11 @@
 (ns nextjournal.clojure-mode.demo
   (:require ["@codemirror/commands" :refer [history historyKeymap]]
-            ["@codemirror/language" :refer [foldGutter syntaxHighlighting defaultHighlightStyle]]
+            ["@codemirror/language" :refer [foldGutter syntaxHighlighting defaultHighlightStyle LRLanguage]]
             ["@codemirror/state" :refer [EditorState]]
             ["@codemirror/view" :as view :refer [EditorView]]
+            ["@lezer/common" :refer [parseMixed]]
+            ["@lezer/generator" :as lg]
+            ["@codemirror/lang-markdown" :as lezer-md]
             ["react" :as react]
             [applied-science.js-interop :as j]
             [clojure.string :as str]
@@ -13,6 +16,7 @@
             [nextjournal.clojure-mode.extensions.eval-region :as eval-region]
             [nextjournal.clojure-mode.keymap :as keymap]
             [nextjournal.clojure-mode.live-grammar :as live-grammar]
+            [nextjournal.clojure-mode.node :as n]
             [nextjournal.clojure-mode.test-utils :as test-utils]
             [nextjournal.livedoc :as livedoc]
             [reagent.core :as r]
@@ -39,6 +43,20 @@
                   ".cm-cursor" {:visibility "hidden"}
                   "&.cm-focused .cm-cursor" {:visibility "visible"}})))
 
+(def parser
+  (lg/buildParser
+   (rc/inline "./clojure.grammar")
+   #js{:externalProp n/node-prop}))
+
+(def mixed-clojure-parser
+  (.configure parser #js {:wrap (parseMixed (fn [node]
+                                              (when (= (.-name node) "LineComment") lezer-md/parser)))}))
+(def mixed-clojure
+  (.define LRLanguage #js {:parser mixed-clojure-parser}))
+
+(comment
+  (js/console.log (.toString (.parse mixed-clojure-parser sample-source))))
+
 (defonce extensions #js[theme
                         (history)
                         (syntaxHighlighting defaultHighlightStyle)
@@ -102,6 +120,7 @@
                                                                                       (seq extensions)
                                                                                       (concat extensions))))))))))}])
 
+
 (defn samples []
   (into [:<>]
         (for [source ["(comment
@@ -112,6 +131,12 @@
   (fizz-buzz 17)
   (fizz-buzz 42))
 
+;; # Here is a Markdown comment
+;; with [links](https://clerk.vision) and *boldness** and *emphasis*;
+;; and with lists, of course:
+;; * Buy milk,
+;; * then throw it away.
+
 (defn fizz-buzz [n]
   (condp (fn [a b] (zero? (mod b a))) n
     15 \"fizzbuzz\"
diff --git a/src/nextjournal/clojure_mode.cljs b/src/nextjournal/clojure_mode.cljs
index f3ebf128..12e714d0 100644
--- a/src/nextjournal/clojure_mode.cljs
+++ b/src/nextjournal/clojure_mode.cljs
@@ -1,7 +1,9 @@
 (ns nextjournal.clojure-mode
   (:require ["@lezer/highlight" :as highlight :refer [tags]]
+            ["@lezer/common" :refer [parseMixed]]
             ["@codemirror/language" :as language :refer [LRLanguage LanguageSupport]]
             ["@nextjournal/lezer-clojure" :as lezer-clj]
+            ["@codemirror/lang-markdown" :as lezer-md]
             [applied-science.js-interop :as j]
             [nextjournal.clojure-mode.extensions.close-brackets :as close-brackets]
             [nextjournal.clojure-mode.extensions.match-brackets :as match-brackets]
@@ -53,7 +55,9 @@
    (.define LRLanguage
             #js {:parser (.configure parser #js {:props #js [format/props
                                                              (.add language/foldNodeProp fold-node-props)
-                                                             (highlight/styleTags style-tags)]})})))
+                                                             (highlight/styleTags style-tags)]
+                                                 :wrap (parseMixed (fn [node]
+                                                                     (when (= (.-name node) "LineComment") lezer-md/parser)))})})))
 
 (def ^js/Array complete-keymap keymap/complete)
 (def ^js/Array builtin-keymap keymap/builtin)

From b2dfbf4bb0a7d0ca36da7ce7e2633d7638dfb6ff Mon Sep 17 00:00:00 2001
From: Andrea Amantini <lo.zampino@gmail.com>
Date: Tue, 27 Jun 2023 16:28:32 +0200
Subject: [PATCH 2/5] Get nested parsing working

---
 demo/src/nextjournal/clojure_mode/demo.cljs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/demo/src/nextjournal/clojure_mode/demo.cljs b/demo/src/nextjournal/clojure_mode/demo.cljs
index ff7024cd..ac725f64 100644
--- a/demo/src/nextjournal/clojure_mode/demo.cljs
+++ b/demo/src/nextjournal/clojure_mode/demo.cljs
@@ -5,7 +5,7 @@
             ["@codemirror/view" :as view :refer [EditorView]]
             ["@lezer/common" :refer [parseMixed]]
             ["@lezer/generator" :as lg]
-            ["@codemirror/lang-markdown" :as lezer-md]
+            ["@codemirror/lang-markdown" :as lang-markdown]
             ["react" :as react]
             [applied-science.js-interop :as j]
             [clojure.string :as str]
@@ -50,12 +50,17 @@
 
 (def mixed-clojure-parser
   (.configure parser #js {:wrap (parseMixed (fn [node]
-                                              (when (= (.-name node) "LineComment") lezer-md/parser)))}))
+                                              (if (= (.-name node) "LineComment")
+                                                (do (js/console.log :node node)
+                                                    (.-markdownLanguage lang-markdown))
+                                                nil)))}))
 (def mixed-clojure
   (.define LRLanguage #js {:parser mixed-clojure-parser}))
 
 (comment
-  (js/console.log (.toString (.parse mixed-clojure-parser sample-source))))
+  (js/console.log (.toString (.parse mixed-clojure-parser ";; # Hello
+(def x 1)
+;; some [nice](link)"))))
 
 (defonce extensions #js[theme
                         (history)

From 4b9d69b5bd3488d349bd43b748b8bee4e00c19d8 Mon Sep 17 00:00:00 2001
From: Andrea Amantini <lo.zampino@gmail.com>
Date: Tue, 27 Jun 2023 17:45:12 +0200
Subject: [PATCH 3/5] Tokenize line comment body

---
 demo/src/nextjournal/clojure_mode/demo.cljs  | 23 ++++++++++++++++----
 src/nextjournal/clojure_mode/clojure.grammar |  4 +++-
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/demo/src/nextjournal/clojure_mode/demo.cljs b/demo/src/nextjournal/clojure_mode/demo.cljs
index ac725f64..34df66e9 100644
--- a/demo/src/nextjournal/clojure_mode/demo.cljs
+++ b/demo/src/nextjournal/clojure_mode/demo.cljs
@@ -48,9 +48,26 @@
    (rc/inline "./clojure.grammar")
    #js{:externalProp n/node-prop}))
 
+(def sample-code
+  ";; # Hello
+(def x 1)
+;; some [nice](link)")
+
+(def tree (.parse parser sample-code))
+
+(comment
+  (js/console.log :plain-clojure (.toString tree))
+
+  (.iterate (n/cursor tree)
+            (fn [node]
+              (js/console.log :node (n/name node) node)
+              true)))
+
+
+
 (def mixed-clojure-parser
   (.configure parser #js {:wrap (parseMixed (fn [node]
-                                              (if (= (.-name node) "LineComment")
+                                              (if (= (.-name node) "CommentText")
                                                 (do (js/console.log :node node)
                                                     (.-markdownLanguage lang-markdown))
                                                 nil)))}))
@@ -58,9 +75,7 @@
   (.define LRLanguage #js {:parser mixed-clojure-parser}))
 
 (comment
-  (js/console.log (.toString (.parse mixed-clojure-parser ";; # Hello
-(def x 1)
-;; some [nice](link)"))))
+  (js/console.log :mixed-clojure (.toString (.parse mixed-clojure-parser sample-code))))
 
 (defonce extensions #js[theme
                         (history)
diff --git a/src/nextjournal/clojure_mode/clojure.grammar b/src/nextjournal/clojure_mode/clojure.grammar
index 4e976bf8..df8a5c87 100644
--- a/src/nextjournal/clojure_mode/clojure.grammar
+++ b/src/nextjournal/clojure_mode/clojure.grammar
@@ -41,6 +41,8 @@ VarName { Symbol }
   ReaderMetadata[prefixColl] { "#^" expression }
   Metadata[prefixColl] { "^" expression }
   String { '"' StringContent? '"' }
+
+  LineComment { ';;' CommentText? }
 }
 
 Meta[prefixContainer]<t> { (Metadata | ReaderMetadata) !meta t }
@@ -90,7 +92,7 @@ Operator { !operator Symbol }
 
   whitespace { (std.whitespace | ",")+ }
 
-  LineComment { ";" ![\n]* }
+  CommentText { ![\n]+ }
 
   // https://docs.oracle.com/javase/specs/jls/se7/html/jls-3.html#jls-3.8
   // class or constructor names

From 6d693c5490116e97650129d02b16bcd0bff06e73 Mon Sep 17 00:00:00 2001
From: Andrea Amantini <lo.zampino@gmail.com>
Date: Wed, 28 Jun 2023 09:34:27 +0200
Subject: [PATCH 4/5] One or two semicolons

---
 src/nextjournal/clojure_mode/clojure.grammar | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nextjournal/clojure_mode/clojure.grammar b/src/nextjournal/clojure_mode/clojure.grammar
index df8a5c87..3f6c9b3e 100644
--- a/src/nextjournal/clojure_mode/clojure.grammar
+++ b/src/nextjournal/clojure_mode/clojure.grammar
@@ -42,7 +42,7 @@ VarName { Symbol }
   Metadata[prefixColl] { "^" expression }
   String { '"' StringContent? '"' }
 
-  LineComment { ';;' CommentText? }
+  LineComment { (';' | ';;') CommentText? }
 }
 
 Meta[prefixContainer]<t> { (Metadata | ReaderMetadata) !meta t }

From 75276b1973f015e6fba2b948c22db910665b1a57 Mon Sep 17 00:00:00 2001
From: Andrea Amantini <lo.zampino@gmail.com>
Date: Wed, 28 Jun 2023 10:03:04 +0200
Subject: [PATCH 5/5] Mixed parsing working

---
 demo/src/nextjournal/clojure_mode/demo.cljs | 38 ++-------------------
 src/nextjournal/clojure_mode.cljs           | 26 +++++++++++---
 2 files changed, 23 insertions(+), 41 deletions(-)

diff --git a/demo/src/nextjournal/clojure_mode/demo.cljs b/demo/src/nextjournal/clojure_mode/demo.cljs
index 34df66e9..e92adc44 100644
--- a/demo/src/nextjournal/clojure_mode/demo.cljs
+++ b/demo/src/nextjournal/clojure_mode/demo.cljs
@@ -5,7 +5,6 @@
             ["@codemirror/view" :as view :refer [EditorView]]
             ["@lezer/common" :refer [parseMixed]]
             ["@lezer/generator" :as lg]
-            ["@codemirror/lang-markdown" :as lang-markdown]
             ["react" :as react]
             [applied-science.js-interop :as j]
             [clojure.string :as str]
@@ -43,40 +42,6 @@
                   ".cm-cursor" {:visibility "hidden"}
                   "&.cm-focused .cm-cursor" {:visibility "visible"}})))
 
-(def parser
-  (lg/buildParser
-   (rc/inline "./clojure.grammar")
-   #js{:externalProp n/node-prop}))
-
-(def sample-code
-  ";; # Hello
-(def x 1)
-;; some [nice](link)")
-
-(def tree (.parse parser sample-code))
-
-(comment
-  (js/console.log :plain-clojure (.toString tree))
-
-  (.iterate (n/cursor tree)
-            (fn [node]
-              (js/console.log :node (n/name node) node)
-              true)))
-
-
-
-(def mixed-clojure-parser
-  (.configure parser #js {:wrap (parseMixed (fn [node]
-                                              (if (= (.-name node) "CommentText")
-                                                (do (js/console.log :node node)
-                                                    (.-markdownLanguage lang-markdown))
-                                                nil)))}))
-(def mixed-clojure
-  (.define LRLanguage #js {:parser mixed-clojure-parser}))
-
-(comment
-  (js/console.log :mixed-clojure (.toString (.parse mixed-clojure-parser sample-code))))
-
 (defonce extensions #js[theme
                         (history)
                         (syntaxHighlighting defaultHighlightStyle)
@@ -87,7 +52,8 @@
                           ;; use live-reloading grammar
                           #js[(cm-clj/syntax live-grammar/parser)
                               (.slice cm-clj/default-extensions 1)]
-                          cm-clj/default-extensions)
+                          cm-clj/mixed-parsing-default-extensions)
+
                         (.of view/keymap cm-clj/complete-keymap)
                         (.of view/keymap historyKeymap)])
 
diff --git a/src/nextjournal/clojure_mode.cljs b/src/nextjournal/clojure_mode.cljs
index 12e714d0..7aedc5af 100644
--- a/src/nextjournal/clojure_mode.cljs
+++ b/src/nextjournal/clojure_mode.cljs
@@ -1,14 +1,15 @@
 (ns nextjournal.clojure-mode
   (:require ["@lezer/highlight" :as highlight :refer [tags]]
-            ["@lezer/common" :refer [parseMixed]]
             ["@codemirror/language" :as language :refer [LRLanguage LanguageSupport]]
+            ["@codemirror/lang-markdown" :as lang-markdown]
             ["@nextjournal/lezer-clojure" :as lezer-clj]
-            ["@codemirror/lang-markdown" :as lezer-md]
+            ["@lezer/common" :refer [parseMixed]]
             [applied-science.js-interop :as j]
             [nextjournal.clojure-mode.extensions.close-brackets :as close-brackets]
             [nextjournal.clojure-mode.extensions.match-brackets :as match-brackets]
             [nextjournal.clojure-mode.extensions.formatting :as format]
             [nextjournal.clojure-mode.extensions.selection-history :as sel-history]
+            [nextjournal.clojure-mode.live-grammar :as live-grammar]
             [nextjournal.clojure-mode.keymap :as keymap]
             [nextjournal.clojure-mode.node :as n]
             [nextjournal.clojure-mode.test-utils :as test-utils]))
@@ -55,9 +56,7 @@
    (.define LRLanguage
             #js {:parser (.configure parser #js {:props #js [format/props
                                                              (.add language/foldNodeProp fold-node-props)
-                                                             (highlight/styleTags style-tags)]
-                                                 :wrap (parseMixed (fn [node]
-                                                                     (when (= (.-name node) "LineComment") lezer-md/parser)))})})))
+                                                             (highlight/styleTags style-tags)]})})))
 
 (def ^js/Array complete-keymap keymap/complete)
 (def ^js/Array builtin-keymap keymap/builtin)
@@ -70,6 +69,23 @@
       (sel-history/extension)
       (format/ext-format-changed-lines)])
 
+(def mixed-parser
+  ;; TODO: fit changes in live grammar into lezer-clojure
+  (.configure live-grammar/parser
+              #js {:wrap (parseMixed (fn [node]
+                                       (when (= (.-name node) "CommentText")
+                                         (.-markdownLanguage lang-markdown))))}))
+
+(def mixed-parsing-default-extensions
+  #js[(syntax mixed-parser)
+      (.slice default-extensions 1)])
+
+(comment
+  (js/console.log :mixed-clojure (.toString (.parse mixed-parser ";; # Helo
+(def x \"x\")
+;; this [link](/foo/bar) **should** be markdown
+"))))
+
 (def language-support
   "Eases embedding clojure mode into other languages (e.g. markdown).
   See https://codemirror.net/docs/ref/#language.LanguageSupport for motivations"
