parse in batches and via idle timers to avoid blocking emacs

2 files changed, 161 insertions, 103 deletions

diff --git a/elfeed-custom-feeds-github.el b/elfeed-custom-feeds-github.el
index a857963..5bfdb6c 100644
--- a/elfeed-custom-feeds-github.el
+++ b/elfeed-custom-feeds-github.el
@@ -50,58 +50,83 @@
                                        (message "Error fetching GitHub PRs %s [%s]: %s" r dn (plist-get status :error))
                                      (goto-char (point-min))
                                      (re-search-forward "\n\n")
-                                     (let* ((json-object-type 'alist)
-                                            (json-array-type 'list)
-                                            (response (json-read))
-                                            (items (alist-get 'items response)))
-                                       (funcall cb r dn items))))
+                                     ;; Capture JSON text for async parsing
+                                     (let ((json-text (buffer-substring-no-properties (point) (point-max))))
+                                       ;; Parse JSON in idle timer to avoid blocking
+                                       (run-with-idle-timer 0 nil
+                                                            (lambda ()
+                                                              (condition-case err
+                                                                  (with-temp-buffer
+                                                                    (insert json-text)
+                                                                    (goto-char (point-min))
+                                                                    (let* ((json-object-type 'alist)
+                                                                           (json-array-type 'list)
+                                                                           (response (json-read))
+                                                                           (items (alist-get 'items response)))
+                                                                      (funcall cb r dn items)))
+                                                                (error
+                                                                 (message "Error parsing JSON for GitHub %s [%s]: %s" r dn err)))))))
                                  nil t)))))
 
 (defun elfeed-custom-feeds-github--prs-to-elfeed (repo display-name prs completion-callback)
   "Convert GitHub PRS to elfeed entries for REPO with DISPLAY-NAME.
 Call COMPLETION-CALLBACK with (repo display-name new-entries) when done."
   (let ((feed-url (format "https://github.com/%s/pulls" repo))
-        (new-entries ()))
-    (let ((feed (elfeed-db-get-feed feed-url)))
-      (setf (elfeed-feed-url feed) feed-url
-            (elfeed-feed-title feed) (format "GitHub: %s [%s]" repo display-name)))
-
-    (dolist (pr prs)
-      (let* ((number (alist-get 'number pr))
-             (title-text (alist-get 'title pr))
-             (html-url (alist-get 'html_url pr))
-             (state (alist-get 'state pr))
-             (updated (alist-get 'updated_at pr))
-             (created (alist-get 'created_at pr))
-             (user (alist-get 'login (alist-get 'user pr)))
-             (body (or (alist-get 'body pr) "No description provided."))
-             (labels (mapcar (lambda (label) (alist-get 'name label))
-                             (alist-get 'labels pr)))
-             (entry-id html-url)
-             (title (format "#%d: %s" number title-text))
-             (date (float-time (date-to-time updated)))
-             (content (format "<h2>PR #%d: %s</h2>\n<p><strong>State:</strong> %s</p>\n<p><strong>Author:</strong> %s</p>\n<p><strong>Created:</strong> %s</p>\n<p><strong>Updated:</strong> %s</p>\n%s\n<h3>Description:</h3>\n<pre>%s</pre>"
-                              number title-text state user created updated
-                              (if labels
-                                  (format "<p><strong>Labels:</strong> %s</p>"
-                                          (mapconcat 'identity labels ", "))
-                                "")
-                              (replace-regexp-in-string "<" "&lt;"
-                                                        (replace-regexp-in-string ">" "&gt;" body)))))
-        (unless (elfeed-db-get-entry entry-id)
-          (let ((entry (elfeed-entry--create
-                        :id entry-id
-                        :title title
-                        :link html-url
-                        :date date
-                        :content content
-                        :content-type 'html
-                        :feed-id feed-url
-                        :tags '(unread github))))
-            (push entry new-entries)))))
-
-    ;; Call completion callback with results
-    (funcall completion-callback repo display-name (nreverse new-entries))))
+        (new-entries ())
+        (prs-to-process prs))
+    ;; Set up feed metadata asynchronously
+    (run-with-idle-timer 0 nil
+                         (lambda ()
+                           (let ((feed (elfeed-db-get-feed feed-url)))
+                             (setf (elfeed-feed-url feed) feed-url
+                                   (elfeed-feed-title feed) (format "GitHub: %s [%s]" repo display-name)))))
+
+    ;; Process PRs in chunks to avoid blocking
+    (cl-labels ((process-next-chunk ()
+                  (when prs-to-process
+                    ;; Process 5 PRs at a time
+                    (let ((chunk (cl-subseq prs-to-process 0 (min 5 (length prs-to-process)))))
+                      (setq prs-to-process (cl-subseq prs-to-process (length chunk)))
+                      ;; Process this chunk
+                      (dolist (pr chunk)
+                        (let* ((number (alist-get 'number pr))
+                               (title-text (alist-get 'title pr))
+                               (html-url (alist-get 'html_url pr))
+                               (state (alist-get 'state pr))
+                               (updated (alist-get 'updated_at pr))
+                               (created (alist-get 'created_at pr))
+                               (user (alist-get 'login (alist-get 'user pr)))
+                               (body (or (alist-get 'body pr) "No description provided."))
+                               (labels (mapcar (lambda (label) (alist-get 'name label))
+                                               (alist-get 'labels pr)))
+                               (entry-id html-url)
+                               (title (format "#%d: %s" number title-text))
+                               (date (float-time (date-to-time updated)))
+                               (content (format "<h2>PR #%d: %s</h2>\n<p><strong>State:</strong> %s</p>\n<p><strong>Author:</strong> %s</p>\n<p><strong>Created:</strong> %s</p>\n<p><strong>Updated:</strong> %s</p>\n%s\n<h3>Description:</h3>\n<pre>%s</pre>"
+                                                number title-text state user created updated
+                                                (if labels
+                                                    (format "<p><strong>Labels:</strong> %s</p>"
+                                                            (mapconcat 'identity labels ", "))
+                                                  "")
+                                                (replace-regexp-in-string "<" "&lt;"
+                                                                          (replace-regexp-in-string ">" "&gt;" body)))))
+                          (unless (elfeed-db-get-entry entry-id)
+                            (let ((entry (elfeed-entry--create
+                                          :id entry-id
+                                          :title title
+                                          :link html-url
+                                          :date date
+                                          :content content
+                                          :content-type 'html
+                                          :feed-id feed-url
+                                          :tags '(unread github))))
+                              (push entry new-entries)))))
+                      ;; Schedule next chunk or call completion
+                      (if prs-to-process
+                          (run-with-idle-timer 0 nil #'process-next-chunk)
+                        ;; All done - call completion callback
+                        (funcall completion-callback repo display-name (nreverse new-entries)))))))
+      (process-next-chunk))))
 
 ;;;###autoload
 (defun elfeed-custom-feeds-github-update ()
@@ -120,26 +145,30 @@ Batches all entries and updates UI once at the end to avoid blocking."
     ;; Completion callback that batches everything
     (let ((completion-callback
            (lambda (repo display-name new-entries)
-             (setq processed (1+ processed))
-             (when new-entries
-               (setq all-new-entries (append all-new-entries new-entries)))
-             (push (list repo display-name (length new-entries)) results)
-
-             ;; When all queries are done, batch add to DB and update UI once
-             (when (= processed total)
-               (let ((total-count (length all-new-entries)))
-                 (when (> total-count 0)
-                   (elfeed-db-add all-new-entries))
-                 ;; Report results for each feed
-                 (dolist (result (nreverse results))
-                   (message "GitHub feed %s [%s]: %d new entries"
-                            (nth 0 result) (nth 1 result) (nth 2 result)))
-                 ;; Defer UI update to avoid blocking
-                 (run-with-idle-timer 0.1 nil
-                                      (lambda ()
-                                        (when (get-buffer "*elfeed-search*")
-                                          (with-current-buffer "*elfeed-search*"
-                                            (elfeed-search-update--force))))))))))
+             (run-with-idle-timer 0 nil
+                                  (lambda ()
+                                    (setq processed (1+ processed))
+                                    (when new-entries
+                                      (setq all-new-entries (append all-new-entries new-entries)))
+                                    (push (list repo display-name (length new-entries)) results)
+
+                                    ;; When all queries are done, batch add to DB and update UI once
+                                    (when (= processed total)
+                                      (let ((total-count (length all-new-entries)))
+                                        (when (> total-count 0)
+                                          (run-with-idle-timer 0 nil
+                                                               (lambda ()
+                                                                 (elfeed-db-add all-new-entries))))
+                                        ;; Report results for each feed
+                                        (dolist (result (nreverse results))
+                                          (message "GitHub feed %s [%s]: %d new entries"
+                                                   (nth 0 result) (nth 1 result) (nth 2 result)))
+                                        ;; Defer UI update to avoid blocking
+                                        (run-with-idle-timer 0.2 nil
+                                                             (lambda ()
+                                                               (when (get-buffer "*elfeed-search*")
+                                                                 (with-current-buffer "*elfeed-search*"
+                                                                   (elfeed-search-update--force)))))))))))
 
       ;; Start all async fetches
       (cl-loop for (repo query display-name) in elfeed-custom-github-pr-queries
diff --git a/elfeed-custom-feeds-huggingface.el b/elfeed-custom-feeds-huggingface.el
index 43f64c8..81f4b5b 100644
--- a/elfeed-custom-feeds-huggingface.el
+++ b/elfeed-custom-feeds-huggingface.el
@@ -47,22 +47,33 @@
                    (url-retrieve url
                                  (lambda (status)
                                    (if (plist-get status :error)
-                                       (funcall cb nil)
+                                       (run-with-idle-timer 0 nil cb nil)
                                      (goto-char (point-min))
                                      (if (re-search-forward "\n\n" nil t)
-                                         (let* ((html (libxml-parse-html-region (point) (point-max)))
-                                                (content-div (or (dom-by-class html "model-card-content")
-                                                                 (dom-by-class html "prose")
-                                                                 (dom-by-tag html 'article))))
-                                           (if content-div
-                                               (with-temp-buffer
-                                                 (dom-print (if (listp content-div) 
-                                                                (car content-div) 
-                                                              content-div))
-                                                 (let ((html-string (buffer-string)))
-                                                   (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string))))
-                                             (funcall cb nil)))
-                                       (funcall cb nil))))
+                                         ;; Capture raw HTML for async processing
+                                         (let ((html-text (buffer-substring-no-properties (point) (point-max))))
+                                           ;; Process HTML parsing in idle timer to avoid blocking
+                                           (run-with-idle-timer 0 nil
+                                                                (lambda ()
+                                                                  (condition-case err
+                                                                      (with-temp-buffer
+                                                                        (insert html-text)
+                                                                        (let* ((html (libxml-parse-html-region (point-min) (point-max)))
+                                                                               (content-div (or (dom-by-class html "model-card-content")
+                                                                                                (dom-by-class html "prose")
+                                                                                                (dom-by-tag html 'article))))
+                                                                          (if content-div
+                                                                              (with-temp-buffer
+                                                                                (dom-print (if (listp content-div)
+                                                                                               (car content-div)
+                                                                                             content-div))
+                                                                                (let ((html-string (buffer-string)))
+                                                                                  (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string))))
+                                                                            (funcall cb nil))))
+                                                                    (error
+                                                                     (message "Error parsing HTML for %s: %s" mid err)
+                                                                     (funcall cb nil))))))
+                                       (run-with-idle-timer 0 nil cb nil))))
                                  nil t)))))
 
 (defun elfeed-custom-feeds-huggingface--fetch-models (org-name callback)
@@ -77,10 +88,21 @@
                         (message "Error fetching HF org %s: %s" org (plist-get status :error))
                       (goto-char (point-min))
                       (re-search-forward "\n\n")
-                      (let* ((json-object-type 'alist)
-                             (json-array-type 'list)
-                             (models (json-read)))
-                        (funcall cb org models))))
+                      ;; Capture JSON text for async parsing
+                      (let ((json-text (buffer-substring-no-properties (point) (point-max))))
+                        ;; Parse JSON in idle timer to avoid blocking
+                        (run-with-idle-timer 0 nil
+                                             (lambda ()
+                                               (condition-case err
+                                                   (with-temp-buffer
+                                                     (insert json-text)
+                                                     (goto-char (point-min))
+                                                     (let* ((json-object-type 'alist)
+                                                            (json-array-type 'list)
+                                                            (models (json-read)))
+                                                       (funcall cb org models)))
+                                                 (error
+                                                  (message "Error parsing JSON for HF org %s: %s" org err))))))))
                   nil t)))
 
 (defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html)
@@ -126,9 +148,11 @@ Batches all entries and updates UI once at the end to avoid blocking."
          (new-entries '())
          (processed 0))
     ;; Set up feed metadata
-    (let ((feed (elfeed-db-get-feed feed-url)))
-      (setf (elfeed-feed-url feed) feed-url
-            (elfeed-feed-title feed) (format "HuggingFace: %s" org-name)))
+    (run-with-idle-timer 0 nil
+                         (lambda ()
+                           (let ((feed (elfeed-db-get-feed feed-url)))
+                             (setf (elfeed-feed-url feed) feed-url
+                                   (elfeed-feed-title feed) (format "HuggingFace: %s" org-name)))))
 
     ;; Return immediately if no models
     (when (zerop total)
@@ -144,22 +168,27 @@ Batches all entries and updates UI once at the end to avoid blocking."
                  model-id
                  (let ((m model))
                    (lambda (html)
-                     (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html)))
-                       (when entry
-                         (push entry new-entries)))
-                     (setq processed (1+ processed))
-                     ;; When all are processed, batch add to DB and update UI once
-                     (when (= processed total)
-                       (let ((count (length new-entries)))
-                         (when (> count 0)
-                           (elfeed-db-add (nreverse new-entries)))
-                         (message "HuggingFace feed %s: %d new entries" org-name count)
-                         ;; Defer UI update to avoid blocking
-                         (run-with-idle-timer 0.1 nil
-                                              (lambda ()
-                                                (when (get-buffer "*elfeed-search*")
-                                                  (with-current-buffer "*elfeed-search*"
-                                                    (elfeed-search-update--force)))))))))
+                     ;; Process entry creation in idle timer to avoid blocking
+                     (run-with-idle-timer 0 nil
+                                          (lambda ()
+                                            (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html)))
+                                              (when entry
+                                                (push entry new-entries)))
+                                            (setq processed (1+ processed))
+                                            ;; When all are processed, batch add to DB and update UI once
+                                            (when (= processed total)
+                                              (let ((count (length new-entries)))
+                                                (when (> count 0)
+                                                  (run-with-idle-timer 0 nil
+                                                                       (lambda ()
+                                                                         (elfeed-db-add (nreverse new-entries)))))
+                                                (message "HuggingFace feed %s: %d new entries" org-name count)
+                                                ;; Defer UI update to avoid blocking
+                                                (run-with-idle-timer 0.2 nil
+                                                                     (lambda ()
+                                                                       (when (get-buffer "*elfeed-search*")
+                                                                         (with-current-buffer "*elfeed-search*"
+                                                                           (elfeed-search-update--force)))))))))))
                  delay))
     nil))