attempt to make the plugin more async-y

3 files changed, 106 insertions, 56 deletions

diff --git a/elfeed-custom-feeds-github.el b/elfeed-custom-feeds-github.el
index 660350d..a857963 100644
--- a/elfeed-custom-feeds-github.el
+++ b/elfeed-custom-feeds-github.el
@@ -57,15 +57,15 @@
                                        (funcall cb r dn items))))
                                  nil t)))))
 
-(defun elfeed-custom-feeds-github--prs-to-elfeed (repo display-name prs)
-  "Convert GitHub PRS to elfeed entries for REPO with DISPLAY-NAME."
+(defun elfeed-custom-feeds-github--prs-to-elfeed (repo display-name prs completion-callback)
+  "Convert GitHub PRS to elfeed entries for REPO with DISPLAY-NAME.
+Call COMPLETION-CALLBACK with (repo display-name new-entries) when done."
   (let ((feed-url (format "https://github.com/%s/pulls" repo))
-        (new-count 0)
         (new-entries ()))
     (let ((feed (elfeed-db-get-feed feed-url)))
       (setf (elfeed-feed-url feed) feed-url
             (elfeed-feed-title feed) (format "GitHub: %s [%s]" repo display-name)))
-    
+
     (dolist (pr prs)
       (let* ((number (alist-get 'number pr))
              (title-text (alist-get 'title pr))
@@ -86,7 +86,7 @@
                                   (format "<p><strong>Labels:</strong> %s</p>"
                                           (mapconcat 'identity labels ", "))
                                 "")
-                              (replace-regexp-in-string "<" "&lt;" 
+                              (replace-regexp-in-string "<" "&lt;"
                                                         (replace-regexp-in-string ">" "&gt;" body)))))
         (unless (elfeed-db-get-entry entry-id)
           (let ((entry (elfeed-entry--create
@@ -98,27 +98,59 @@
                         :content-type 'html
                         :feed-id feed-url
                         :tags '(unread github))))
-            (push entry new-entries)
-            (setq new-count (1+ new-count))))))
-    
-    (when new-entries
-      (elfeed-db-add (nreverse new-entries)))
-    
-    (message "GitHub feed %s [%s]: %d new entries" repo display-name new-count)
-    (when (get-buffer "*elfeed-search*")
-      (with-current-buffer "*elfeed-search*"
-        (elfeed-search-update--force)))))
+            (push entry new-entries)))))
+
+    ;; Call completion callback with results
+    (funcall completion-callback repo display-name (nreverse new-entries))))
 
 ;;;###autoload
 (defun elfeed-custom-feeds-github-update ()
-  "Update all GitHub PR feeds."
+  "Update all GitHub PR feeds asynchronously.
+Batches all entries and updates UI once at the end to avoid blocking."
   (interactive)
-  (cl-loop for (repo query display-name) in elfeed-custom-github-pr-queries
-           for i from 0
-           for delay = (* i elfeed-custom-github-request-delay)
-           do (elfeed-custom-feeds-github--fetch-prs repo query display-name
-                                                     #'elfeed-custom-feeds-github--prs-to-elfeed 
-                                                     delay)))
+  (let* ((total (length elfeed-custom-github-pr-queries))
+         (processed 0)
+         (all-new-entries '())
+         (results '()))
+
+    (when (zerop total)
+      (message "No GitHub PR queries configured")
+      (cl-return-from elfeed-custom-feeds-github-update nil))
+
+    ;; Completion callback that batches everything
+    (let ((completion-callback
+           (lambda (repo display-name new-entries)
+             (setq processed (1+ processed))
+             (when new-entries
+               (setq all-new-entries (append all-new-entries new-entries)))
+             (push (list repo display-name (length new-entries)) results)
+
+             ;; When all queries are done, batch add to DB and update UI once
+             (when (= processed total)
+               (let ((total-count (length all-new-entries)))
+                 (when (> total-count 0)
+                   (elfeed-db-add all-new-entries))
+                 ;; Report results for each feed
+                 (dolist (result (nreverse results))
+                   (message "GitHub feed %s [%s]: %d new entries"
+                            (nth 0 result) (nth 1 result) (nth 2 result)))
+                 ;; Defer UI update to avoid blocking
+                 (run-with-idle-timer 0.1 nil
+                                      (lambda ()
+                                        (when (get-buffer "*elfeed-search*")
+                                          (with-current-buffer "*elfeed-search*"
+                                            (elfeed-search-update--force))))))))))
+
+      ;; Start all async fetches
+      (cl-loop for (repo query display-name) in elfeed-custom-github-pr-queries
+               for i from 0
+               for delay = (* i elfeed-custom-github-request-delay)
+               do (elfeed-custom-feeds-github--fetch-prs
+                   repo query display-name
+                   (lambda (r dn items)
+                     (elfeed-custom-feeds-github--prs-to-elfeed r dn items completion-callback))
+                   delay)))
+    nil))
 
 (provide 'elfeed-custom-feeds-github)
 
diff --git a/elfeed-custom-feeds-huggingface.el b/elfeed-custom-feeds-huggingface.el
index a3bfdb0..43f64c8 100644
--- a/elfeed-custom-feeds-huggingface.el
+++ b/elfeed-custom-feeds-huggingface.el
@@ -84,7 +84,8 @@
                   nil t)))
 
 (defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html)
-  "Create an elfeed entry for MODEL with model card HTML content."
+  "Create an elfeed entry for MODEL with model card HTML content.
+Returns the entry if it's new, nil otherwise."
   (let* ((feed-url (format "https://huggingface.co/%s" org-name))
          (model-id (alist-get 'id model))
          (last-modified (alist-get 'lastModified model))
@@ -96,64 +97,80 @@
          (title (format "[%s] %s" org-name model-id))
          (date (float-time (date-to-time last-modified)))
          (metadata (format "<h2>%s</h2>\n<p><strong>Type:</strong> %s</p>\n<p><strong>Downloads:</strong> %d</p>\n<p><strong>Last Modified:</strong> %s</p>\n%s\n<hr>\n"
-                           model-id 
-                           pipeline 
-                           downloads 
+                           model-id
+                           pipeline
+                           downloads
                            last-modified
                            (if tags
                                (format "<p><strong>Tags:</strong> %s</p>"
                                        (mapconcat 'identity (cl-subseq tags 0 (min 10 (length tags))) ", "))
                              "")))
-         (content (concat metadata 
+         (content (concat metadata
                           (or model-card-html "<p><em>Model card not available</em></p>"))))
     (unless (elfeed-db-get-entry entry-id)
-      (let ((entry (elfeed-entry--create
-                    :id entry-id
-                    :title title
-                    :link link
-                    :date date
-                    :content content
-                    :content-type 'html
-                    :feed-id feed-url
-                    :tags '(unread huggingface))))
-        (elfeed-db-add (list entry))
-        (when (get-buffer "*elfeed-search*")
-          (with-current-buffer "*elfeed-search*"
-            (elfeed-search-update--force)))
-        entry))))
+      (elfeed-entry--create
+       :id entry-id
+       :title title
+       :link link
+       :date date
+       :content content
+       :content-type 'html
+       :feed-id feed-url
+       :tags '(unread huggingface)))))
 
 (defun elfeed-custom-feeds-huggingface--models-to-elfeed (org-name models)
-  "Convert HuggingFace MODELS to elfeed entries for ORG-NAME."
-  (let ((feed-url (format "https://huggingface.co/%s" org-name))
-        (new-count 0)
-        (total (length models))
-        (org org-name))
+  "Convert HuggingFace MODELS to elfeed entries for ORG-NAME.
+Batches all entries and updates UI once at the end to avoid blocking."
+  (let* ((feed-url (format "https://huggingface.co/%s" org-name))
+         (total (length models))
+         (new-entries '())
+         (processed 0))
+    ;; Set up feed metadata
     (let ((feed (elfeed-db-get-feed feed-url)))
       (setf (elfeed-feed-url feed) feed-url
             (elfeed-feed-title feed) (format "HuggingFace: %s" org-name)))
-    
+
+    ;; Return immediately if no models
+    (when (zerop total)
+      (message "HuggingFace feed %s: no models to process" org-name)
+      (cl-return-from elfeed-custom-feeds-huggingface--models-to-elfeed nil))
+
+    ;; Fetch model cards asynchronously and batch entries
     (cl-loop for model in models
              for i from 0
              for model-id = (alist-get 'id model)
              for delay = (* i elfeed-custom-huggingface-request-delay)
-             do (elfeed-custom-feeds-huggingface--fetch-model-card 
+             do (elfeed-custom-feeds-huggingface--fetch-model-card
                  model-id
                  (let ((m model))
                    (lambda (html)
-                     (when (elfeed-custom-feeds-huggingface--create-entry org m html)
-                       (setq new-count (1+ new-count)))
-                     (when (= new-count total)
-                       (message "HuggingFace feed %s: %d new entries" org new-count))))
+                     (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html)))
+                       (when entry
+                         (push entry new-entries)))
+                     (setq processed (1+ processed))
+                     ;; When all are processed, batch add to DB and update UI once
+                     (when (= processed total)
+                       (let ((count (length new-entries)))
+                         (when (> count 0)
+                           (elfeed-db-add (nreverse new-entries)))
+                         (message "HuggingFace feed %s: %d new entries" org-name count)
+                         ;; Defer UI update to avoid blocking
+                         (run-with-idle-timer 0.1 nil
+                                              (lambda ()
+                                                (when (get-buffer "*elfeed-search*")
+                                                  (with-current-buffer "*elfeed-search*"
+                                                    (elfeed-search-update--force)))))))))
                  delay))
-    
     nil))
 
 ;;;###autoload
 (defun elfeed-custom-feeds-huggingface-update ()
-  "Update all HuggingFace organization feeds."
+  "Update all HuggingFace organization feeds asynchronously.
+Returns immediately while updates happen in the background."
   (interactive)
   (dolist (org elfeed-custom-huggingface-orgs)
-    (elfeed-custom-feeds-huggingface--fetch-models org #'elfeed-custom-feeds-huggingface--models-to-elfeed)))
+    (elfeed-custom-feeds-huggingface--fetch-models org #'elfeed-custom-feeds-huggingface--models-to-elfeed))
+  nil)
 
 (provide 'elfeed-custom-feeds-huggingface)
 
diff --git a/elfeed-custom-feeds.el b/elfeed-custom-feeds.el
index a64cc69..0e8ff6e 100644
--- a/elfeed-custom-feeds.el
+++ b/elfeed-custom-feeds.el
@@ -31,7 +31,8 @@
   (interactive)
   (elfeed-custom-feeds-huggingface-update)
   (elfeed-custom-feeds-github-update)
-  (message "Updating custom feeds..."))
+  (message "Updating custom feeds...")
+  nil)
 
 (provide 'elfeed-custom-feeds)