From af9627b43eaf4e40fadc9be82ccc2f10f33746a3 Mon Sep 17 00:00:00 2001 From: vin Date: Mon, 8 Dec 2025 16:51:59 -0500 Subject: attempt to make the plugin more async-y --- elfeed-custom-feeds-github.el | 76 ++++++++++++++++++++++++---------- elfeed-custom-feeds-huggingface.el | 83 +++++++++++++++++++++++--------------- elfeed-custom-feeds.el | 3 +- 3 files changed, 106 insertions(+), 56 deletions(-) diff --git a/elfeed-custom-feeds-github.el b/elfeed-custom-feeds-github.el index 660350d..a857963 100644 --- a/elfeed-custom-feeds-github.el +++ b/elfeed-custom-feeds-github.el @@ -57,15 +57,15 @@ (funcall cb r dn items)))) nil t))))) -(defun elfeed-custom-feeds-github--prs-to-elfeed (repo display-name prs) - "Convert GitHub PRS to elfeed entries for REPO with DISPLAY-NAME." +(defun elfeed-custom-feeds-github--prs-to-elfeed (repo display-name prs completion-callback) + "Convert GitHub PRS to elfeed entries for REPO with DISPLAY-NAME. +Call COMPLETION-CALLBACK with (repo display-name new-entries) when done." (let ((feed-url (format "https://github.com/%s/pulls" repo)) - (new-count 0) (new-entries ())) (let ((feed (elfeed-db-get-feed feed-url))) (setf (elfeed-feed-url feed) feed-url (elfeed-feed-title feed) (format "GitHub: %s [%s]" repo display-name))) - + (dolist (pr prs) (let* ((number (alist-get 'number pr)) (title-text (alist-get 'title pr)) @@ -86,7 +86,7 @@ (format "

Labels: %s

" (mapconcat 'identity labels ", ")) "") - (replace-regexp-in-string "<" "<" + (replace-regexp-in-string "<" "<" (replace-regexp-in-string ">" ">" body))))) (unless (elfeed-db-get-entry entry-id) (let ((entry (elfeed-entry--create @@ -98,27 +98,59 @@ :content-type 'html :feed-id feed-url :tags '(unread github)))) - (push entry new-entries) - (setq new-count (1+ new-count)))))) - - (when new-entries - (elfeed-db-add (nreverse new-entries))) - - (message "GitHub feed %s [%s]: %d new entries" repo display-name new-count) - (when (get-buffer "*elfeed-search*") - (with-current-buffer "*elfeed-search*" - (elfeed-search-update--force))))) + (push entry new-entries))))) + + ;; Call completion callback with results + (funcall completion-callback repo display-name (nreverse new-entries)))) ;;;###autoload (defun elfeed-custom-feeds-github-update () - "Update all GitHub PR feeds." + "Update all GitHub PR feeds asynchronously. +Batches all entries and updates UI once at the end to avoid blocking." (interactive) - (cl-loop for (repo query display-name) in elfeed-custom-github-pr-queries - for i from 0 - for delay = (* i elfeed-custom-github-request-delay) - do (elfeed-custom-feeds-github--fetch-prs repo query display-name - #'elfeed-custom-feeds-github--prs-to-elfeed - delay))) + (let* ((total (length elfeed-custom-github-pr-queries)) + (processed 0) + (all-new-entries '()) + (results '())) + + (when (zerop total) + (message "No GitHub PR queries configured") + (cl-return-from elfeed-custom-feeds-github-update nil)) + + ;; Completion callback that batches everything + (let ((completion-callback + (lambda (repo display-name new-entries) + (setq processed (1+ processed)) + (when new-entries + (setq all-new-entries (append all-new-entries new-entries))) + (push (list repo display-name (length new-entries)) results) + + ;; When all queries are done, batch add to DB and update UI once + (when (= processed total) + (let ((total-count (length all-new-entries))) + (when (> total-count 0) + (elfeed-db-add all-new-entries)) + ;; Report results for each feed + (dolist (result (nreverse results)) + (message "GitHub feed %s [%s]: %d new entries" + (nth 0 result) (nth 1 result) (nth 2 result))) + ;; Defer UI update to avoid blocking + (run-with-idle-timer 0.1 nil + (lambda () + (when (get-buffer "*elfeed-search*") + (with-current-buffer "*elfeed-search*" + (elfeed-search-update--force)))))))))) + + ;; Start all async fetches + (cl-loop for (repo query display-name) in elfeed-custom-github-pr-queries + for i from 0 + for delay = (* i elfeed-custom-github-request-delay) + do (elfeed-custom-feeds-github--fetch-prs + repo query display-name + (lambda (r dn items) + (elfeed-custom-feeds-github--prs-to-elfeed r dn items completion-callback)) + delay))) + nil)) (provide 'elfeed-custom-feeds-github) diff --git a/elfeed-custom-feeds-huggingface.el b/elfeed-custom-feeds-huggingface.el index a3bfdb0..43f64c8 100644 --- a/elfeed-custom-feeds-huggingface.el +++ b/elfeed-custom-feeds-huggingface.el @@ -84,7 +84,8 @@ nil t))) (defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html) - "Create an elfeed entry for MODEL with model card HTML content." + "Create an elfeed entry for MODEL with model card HTML content. +Returns the entry if it's new, nil otherwise." (let* ((feed-url (format "https://huggingface.co/%s" org-name)) (model-id (alist-get 'id model)) (last-modified (alist-get 'lastModified model)) @@ -96,64 +97,80 @@ (title (format "[%s] %s" org-name model-id)) (date (float-time (date-to-time last-modified))) (metadata (format "

%s

\n

Type: %s

\n

Downloads: %d

\n

Last Modified: %s

\n%s\n
\n" - model-id - pipeline - downloads + model-id + pipeline + downloads last-modified (if tags (format "

Tags: %s

" (mapconcat 'identity (cl-subseq tags 0 (min 10 (length tags))) ", ")) ""))) - (content (concat metadata + (content (concat metadata (or model-card-html "

Model card not available

")))) (unless (elfeed-db-get-entry entry-id) - (let ((entry (elfeed-entry--create - :id entry-id - :title title - :link link - :date date - :content content - :content-type 'html - :feed-id feed-url - :tags '(unread huggingface)))) - (elfeed-db-add (list entry)) - (when (get-buffer "*elfeed-search*") - (with-current-buffer "*elfeed-search*" - (elfeed-search-update--force))) - entry)))) + (elfeed-entry--create + :id entry-id + :title title + :link link + :date date + :content content + :content-type 'html + :feed-id feed-url + :tags '(unread huggingface))))) (defun elfeed-custom-feeds-huggingface--models-to-elfeed (org-name models) - "Convert HuggingFace MODELS to elfeed entries for ORG-NAME." - (let ((feed-url (format "https://huggingface.co/%s" org-name)) - (new-count 0) - (total (length models)) - (org org-name)) + "Convert HuggingFace MODELS to elfeed entries for ORG-NAME. +Batches all entries and updates UI once at the end to avoid blocking." + (let* ((feed-url (format "https://huggingface.co/%s" org-name)) + (total (length models)) + (new-entries '()) + (processed 0)) + ;; Set up feed metadata (let ((feed (elfeed-db-get-feed feed-url))) (setf (elfeed-feed-url feed) feed-url (elfeed-feed-title feed) (format "HuggingFace: %s" org-name))) - + + ;; Return immediately if no models + (when (zerop total) + (message "HuggingFace feed %s: no models to process" org-name) + (cl-return-from elfeed-custom-feeds-huggingface--models-to-elfeed nil)) + + ;; Fetch model cards asynchronously and batch entries (cl-loop for model in models for i from 0 for model-id = (alist-get 'id model) for delay = (* i elfeed-custom-huggingface-request-delay) - do (elfeed-custom-feeds-huggingface--fetch-model-card + do (elfeed-custom-feeds-huggingface--fetch-model-card model-id (let ((m model)) (lambda (html) - (when (elfeed-custom-feeds-huggingface--create-entry org m html) - (setq new-count (1+ new-count))) - (when (= new-count total) - (message "HuggingFace feed %s: %d new entries" org new-count)))) + (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html))) + (when entry + (push entry new-entries))) + (setq processed (1+ processed)) + ;; When all are processed, batch add to DB and update UI once + (when (= processed total) + (let ((count (length new-entries))) + (when (> count 0) + (elfeed-db-add (nreverse new-entries))) + (message "HuggingFace feed %s: %d new entries" org-name count) + ;; Defer UI update to avoid blocking + (run-with-idle-timer 0.1 nil + (lambda () + (when (get-buffer "*elfeed-search*") + (with-current-buffer "*elfeed-search*" + (elfeed-search-update--force))))))))) delay)) - nil)) ;;;###autoload (defun elfeed-custom-feeds-huggingface-update () - "Update all HuggingFace organization feeds." + "Update all HuggingFace organization feeds asynchronously. +Returns immediately while updates happen in the background." (interactive) (dolist (org elfeed-custom-huggingface-orgs) - (elfeed-custom-feeds-huggingface--fetch-models org #'elfeed-custom-feeds-huggingface--models-to-elfeed))) + (elfeed-custom-feeds-huggingface--fetch-models org #'elfeed-custom-feeds-huggingface--models-to-elfeed)) + nil) (provide 'elfeed-custom-feeds-huggingface) diff --git a/elfeed-custom-feeds.el b/elfeed-custom-feeds.el index a64cc69..0e8ff6e 100644 --- a/elfeed-custom-feeds.el +++ b/elfeed-custom-feeds.el @@ -31,7 +31,8 @@ (interactive) (elfeed-custom-feeds-huggingface-update) (elfeed-custom-feeds-github-update) - (message "Updating custom feeds...")) + (message "Updating custom feeds...") + nil) (provide 'elfeed-custom-feeds) -- cgit v1.2.3