commit b67d770226c84c0cd5869a352622ebbd3b0e7ba4
parent af9627b43eaf4e40fadc9be82ccc2f10f33746a3
Author: vin <git@vineetk.net>
Date: Mon, 8 Dec 2025 17:02:56 -0500
parse in batches and via idle timers to avoid blocking emacs
Diffstat:
2 files changed, 161 insertions(+), 103 deletions(-)
diff --git a/elfeed-custom-feeds-github.el b/elfeed-custom-feeds-github.el
@@ -50,58 +50,83 @@
(message "Error fetching GitHub PRs %s [%s]: %s" r dn (plist-get status :error))
(goto-char (point-min))
(re-search-forward "\n\n")
- (let* ((json-object-type 'alist)
- (json-array-type 'list)
- (response (json-read))
- (items (alist-get 'items response)))
- (funcall cb r dn items))))
+ ;; Capture JSON text for async parsing
+ (let ((json-text (buffer-substring-no-properties (point) (point-max))))
+ ;; Parse JSON in idle timer to avoid blocking
+ (run-with-idle-timer 0 nil
+ (lambda ()
+ (condition-case err
+ (with-temp-buffer
+ (insert json-text)
+ (goto-char (point-min))
+ (let* ((json-object-type 'alist)
+ (json-array-type 'list)
+ (response (json-read))
+ (items (alist-get 'items response)))
+ (funcall cb r dn items)))
+ (error
+ (message "Error parsing JSON for GitHub %s [%s]: %s" r dn err)))))))
nil t)))))
(defun elfeed-custom-feeds-github--prs-to-elfeed (repo display-name prs completion-callback)
"Convert GitHub PRS to elfeed entries for REPO with DISPLAY-NAME.
Call COMPLETION-CALLBACK with (repo display-name new-entries) when done."
(let ((feed-url (format "https://github.com/%s/pulls" repo))
- (new-entries ()))
- (let ((feed (elfeed-db-get-feed feed-url)))
- (setf (elfeed-feed-url feed) feed-url
- (elfeed-feed-title feed) (format "GitHub: %s [%s]" repo display-name)))
-
- (dolist (pr prs)
- (let* ((number (alist-get 'number pr))
- (title-text (alist-get 'title pr))
- (html-url (alist-get 'html_url pr))
- (state (alist-get 'state pr))
- (updated (alist-get 'updated_at pr))
- (created (alist-get 'created_at pr))
- (user (alist-get 'login (alist-get 'user pr)))
- (body (or (alist-get 'body pr) "No description provided."))
- (labels (mapcar (lambda (label) (alist-get 'name label))
- (alist-get 'labels pr)))
- (entry-id html-url)
- (title (format "#%d: %s" number title-text))
- (date (float-time (date-to-time updated)))
- (content (format "<h2>PR #%d: %s</h2>\n<p><strong>State:</strong> %s</p>\n<p><strong>Author:</strong> %s</p>\n<p><strong>Created:</strong> %s</p>\n<p><strong>Updated:</strong> %s</p>\n%s\n<h3>Description:</h3>\n<pre>%s</pre>"
- number title-text state user created updated
- (if labels
- (format "<p><strong>Labels:</strong> %s</p>"
- (mapconcat 'identity labels ", "))
- "")
- (replace-regexp-in-string "<" "<"
- (replace-regexp-in-string ">" ">" body)))))
- (unless (elfeed-db-get-entry entry-id)
- (let ((entry (elfeed-entry--create
- :id entry-id
- :title title
- :link html-url
- :date date
- :content content
- :content-type 'html
- :feed-id feed-url
- :tags '(unread github))))
- (push entry new-entries)))))
-
- ;; Call completion callback with results
- (funcall completion-callback repo display-name (nreverse new-entries))))
+ (new-entries ())
+ (prs-to-process prs))
+ ;; Set up feed metadata asynchronously
+ (run-with-idle-timer 0 nil
+ (lambda ()
+ (let ((feed (elfeed-db-get-feed feed-url)))
+ (setf (elfeed-feed-url feed) feed-url
+ (elfeed-feed-title feed) (format "GitHub: %s [%s]" repo display-name)))))
+
+ ;; Process PRs in chunks to avoid blocking
+ (cl-labels ((process-next-chunk ()
+ (when prs-to-process
+ ;; Process 5 PRs at a time
+ (let ((chunk (cl-subseq prs-to-process 0 (min 5 (length prs-to-process)))))
+ (setq prs-to-process (cl-subseq prs-to-process (length chunk)))
+ ;; Process this chunk
+ (dolist (pr chunk)
+ (let* ((number (alist-get 'number pr))
+ (title-text (alist-get 'title pr))
+ (html-url (alist-get 'html_url pr))
+ (state (alist-get 'state pr))
+ (updated (alist-get 'updated_at pr))
+ (created (alist-get 'created_at pr))
+ (user (alist-get 'login (alist-get 'user pr)))
+ (body (or (alist-get 'body pr) "No description provided."))
+ (labels (mapcar (lambda (label) (alist-get 'name label))
+ (alist-get 'labels pr)))
+ (entry-id html-url)
+ (title (format "#%d: %s" number title-text))
+ (date (float-time (date-to-time updated)))
+ (content (format "<h2>PR #%d: %s</h2>\n<p><strong>State:</strong> %s</p>\n<p><strong>Author:</strong> %s</p>\n<p><strong>Created:</strong> %s</p>\n<p><strong>Updated:</strong> %s</p>\n%s\n<h3>Description:</h3>\n<pre>%s</pre>"
+ number title-text state user created updated
+ (if labels
+ (format "<p><strong>Labels:</strong> %s</p>"
+ (mapconcat 'identity labels ", "))
+ "")
+ (replace-regexp-in-string "<" "<"
+ (replace-regexp-in-string ">" ">" body)))))
+ (unless (elfeed-db-get-entry entry-id)
+ (let ((entry (elfeed-entry--create
+ :id entry-id
+ :title title
+ :link html-url
+ :date date
+ :content content
+ :content-type 'html
+ :feed-id feed-url
+ :tags '(unread github))))
+ (push entry new-entries)))))
+ ;; Schedule next chunk or call completion
+ (if prs-to-process
+ (run-with-idle-timer 0 nil #'process-next-chunk)
+ ;; All done - call completion callback
+ (funcall completion-callback repo display-name (nreverse new-entries)))))))
+ (process-next-chunk))))
;;;###autoload
(defun elfeed-custom-feeds-github-update ()
@@ -120,26 +145,30 @@ Batches all entries and updates UI once at the end to avoid blocking."
;; Completion callback that batches everything
(let ((completion-callback
(lambda (repo display-name new-entries)
- (setq processed (1+ processed))
- (when new-entries
- (setq all-new-entries (append all-new-entries new-entries)))
- (push (list repo display-name (length new-entries)) results)
-
- ;; When all queries are done, batch add to DB and update UI once
- (when (= processed total)
- (let ((total-count (length all-new-entries)))
- (when (> total-count 0)
- (elfeed-db-add all-new-entries))
- ;; Report results for each feed
- (dolist (result (nreverse results))
- (message "GitHub feed %s [%s]: %d new entries"
- (nth 0 result) (nth 1 result) (nth 2 result)))
- ;; Defer UI update to avoid blocking
- (run-with-idle-timer 0.1 nil
- (lambda ()
- (when (get-buffer "*elfeed-search*")
- (with-current-buffer "*elfeed-search*"
- (elfeed-search-update--force))))))))))
+ (run-with-idle-timer 0 nil
+ (lambda ()
+ (setq processed (1+ processed))
+ (when new-entries
+ (setq all-new-entries (append all-new-entries new-entries)))
+ (push (list repo display-name (length new-entries)) results)
+
+ ;; When all queries are done, batch add to DB and update UI once
+ (when (= processed total)
+ (let ((total-count (length all-new-entries)))
+ (when (> total-count 0)
+ (run-with-idle-timer 0 nil
+ (lambda ()
+ (elfeed-db-add all-new-entries))))
+ ;; Report results for each feed
+ (dolist (result (nreverse results))
+ (message "GitHub feed %s [%s]: %d new entries"
+ (nth 0 result) (nth 1 result) (nth 2 result)))
+ ;; Defer UI update to avoid blocking
+ (run-with-idle-timer 0.2 nil
+ (lambda ()
+ (when (get-buffer "*elfeed-search*")
+ (with-current-buffer "*elfeed-search*"
+ (elfeed-search-update--force)))))))))))
;; Start all async fetches
(cl-loop for (repo query display-name) in elfeed-custom-github-pr-queries
diff --git a/elfeed-custom-feeds-huggingface.el b/elfeed-custom-feeds-huggingface.el
@@ -47,22 +47,33 @@
(url-retrieve url
(lambda (status)
(if (plist-get status :error)
- (funcall cb nil)
+ (run-with-idle-timer 0 nil cb nil)
(goto-char (point-min))
(if (re-search-forward "\n\n" nil t)
- (let* ((html (libxml-parse-html-region (point) (point-max)))
- (content-div (or (dom-by-class html "model-card-content")
- (dom-by-class html "prose")
- (dom-by-tag html 'article))))
- (if content-div
- (with-temp-buffer
- (dom-print (if (listp content-div)
- (car content-div)
- content-div))
- (let ((html-string (buffer-string)))
- (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string))))
- (funcall cb nil)))
- (funcall cb nil))))
+ ;; Capture raw HTML for async processing
+ (let ((html-text (buffer-substring-no-properties (point) (point-max))))
+ ;; Process HTML parsing in idle timer to avoid blocking
+ (run-with-idle-timer 0 nil
+ (lambda ()
+ (condition-case err
+ (with-temp-buffer
+ (insert html-text)
+ (let* ((html (libxml-parse-html-region (point-min) (point-max)))
+ (content-div (or (dom-by-class html "model-card-content")
+ (dom-by-class html "prose")
+ (dom-by-tag html 'article))))
+ (if content-div
+ (with-temp-buffer
+ (dom-print (if (listp content-div)
+ (car content-div)
+ content-div))
+ (let ((html-string (buffer-string)))
+ (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string))))
+ (funcall cb nil))))
+ (error
+ (message "Error parsing HTML for %s: %s" mid err)
+ (funcall cb nil))))))
+ (run-with-idle-timer 0 nil cb nil))))
nil t)))))
(defun elfeed-custom-feeds-huggingface--fetch-models (org-name callback)
@@ -77,10 +88,21 @@
(message "Error fetching HF org %s: %s" org (plist-get status :error))
(goto-char (point-min))
(re-search-forward "\n\n")
- (let* ((json-object-type 'alist)
- (json-array-type 'list)
- (models (json-read)))
- (funcall cb org models))))
+ ;; Capture JSON text for async parsing
+ (let ((json-text (buffer-substring-no-properties (point) (point-max))))
+ ;; Parse JSON in idle timer to avoid blocking
+ (run-with-idle-timer 0 nil
+ (lambda ()
+ (condition-case err
+ (with-temp-buffer
+ (insert json-text)
+ (goto-char (point-min))
+ (let* ((json-object-type 'alist)
+ (json-array-type 'list)
+ (models (json-read)))
+ (funcall cb org models)))
+ (error
+ (message "Error parsing JSON for HF org %s: %s" org err))))))))
nil t)))
(defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html)
@@ -126,9 +148,11 @@ Batches all entries and updates UI once at the end to avoid blocking."
(new-entries '())
(processed 0))
;; Set up feed metadata
- (let ((feed (elfeed-db-get-feed feed-url)))
- (setf (elfeed-feed-url feed) feed-url
- (elfeed-feed-title feed) (format "HuggingFace: %s" org-name)))
+ (run-with-idle-timer 0 nil
+ (lambda ()
+ (let ((feed (elfeed-db-get-feed feed-url)))
+ (setf (elfeed-feed-url feed) feed-url
+ (elfeed-feed-title feed) (format "HuggingFace: %s" org-name)))))
;; Return immediately if no models
(when (zerop total)
@@ -144,22 +168,27 @@ Batches all entries and updates UI once at the end to avoid blocking."
model-id
(let ((m model))
(lambda (html)
- (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html)))
- (when entry
- (push entry new-entries)))
- (setq processed (1+ processed))
- ;; When all are processed, batch add to DB and update UI once
- (when (= processed total)
- (let ((count (length new-entries)))
- (when (> count 0)
- (elfeed-db-add (nreverse new-entries)))
- (message "HuggingFace feed %s: %d new entries" org-name count)
- ;; Defer UI update to avoid blocking
- (run-with-idle-timer 0.1 nil
- (lambda ()
- (when (get-buffer "*elfeed-search*")
- (with-current-buffer "*elfeed-search*"
- (elfeed-search-update--force)))))))))
+ ;; Process entry creation in idle timer to avoid blocking
+ (run-with-idle-timer 0 nil
+ (lambda ()
+ (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html)))
+ (when entry
+ (push entry new-entries)))
+ (setq processed (1+ processed))
+ ;; When all are processed, batch add to DB and update UI once
+ (when (= processed total)
+ (let ((count (length new-entries)))
+ (when (> count 0)
+ (run-with-idle-timer 0 nil
+ (lambda ()
+ (elfeed-db-add (nreverse new-entries)))))
+ (message "HuggingFace feed %s: %d new entries" org-name count)
+ ;; Defer UI update to avoid blocking
+ (run-with-idle-timer 0.2 nil
+ (lambda ()
+ (when (get-buffer "*elfeed-search*")
+ (with-current-buffer "*elfeed-search*"
+ (elfeed-search-update--force)))))))))))
delay))
nil))
