From b67d770226c84c0cd5869a352622ebbd3b0e7ba4 Mon Sep 17 00:00:00 2001 From: vin Date: Mon, 8 Dec 2025 17:02:56 -0500 Subject: parse in batches and via idle timers to avoid blocking emacs --- elfeed-custom-feeds-github.el | 161 ++++++++++++++++++++++--------------- elfeed-custom-feeds-huggingface.el | 103 +++++++++++++++--------- 2 files changed, 161 insertions(+), 103 deletions(-) diff --git a/elfeed-custom-feeds-github.el b/elfeed-custom-feeds-github.el index a857963..5bfdb6c 100644 --- a/elfeed-custom-feeds-github.el +++ b/elfeed-custom-feeds-github.el @@ -50,58 +50,83 @@ (message "Error fetching GitHub PRs %s [%s]: %s" r dn (plist-get status :error)) (goto-char (point-min)) (re-search-forward "\n\n") - (let* ((json-object-type 'alist) - (json-array-type 'list) - (response (json-read)) - (items (alist-get 'items response))) - (funcall cb r dn items)))) + ;; Capture JSON text for async parsing + (let ((json-text (buffer-substring-no-properties (point) (point-max)))) + ;; Parse JSON in idle timer to avoid blocking + (run-with-idle-timer 0 nil + (lambda () + (condition-case err + (with-temp-buffer + (insert json-text) + (goto-char (point-min)) + (let* ((json-object-type 'alist) + (json-array-type 'list) + (response (json-read)) + (items (alist-get 'items response))) + (funcall cb r dn items))) + (error + (message "Error parsing JSON for GitHub %s [%s]: %s" r dn err))))))) nil t))))) (defun elfeed-custom-feeds-github--prs-to-elfeed (repo display-name prs completion-callback) "Convert GitHub PRS to elfeed entries for REPO with DISPLAY-NAME. Call COMPLETION-CALLBACK with (repo display-name new-entries) when done." (let ((feed-url (format "https://github.com/%s/pulls" repo)) - (new-entries ())) - (let ((feed (elfeed-db-get-feed feed-url))) - (setf (elfeed-feed-url feed) feed-url - (elfeed-feed-title feed) (format "GitHub: %s [%s]" repo display-name))) - - (dolist (pr prs) - (let* ((number (alist-get 'number pr)) - (title-text (alist-get 'title pr)) - (html-url (alist-get 'html_url pr)) - (state (alist-get 'state pr)) - (updated (alist-get 'updated_at pr)) - (created (alist-get 'created_at pr)) - (user (alist-get 'login (alist-get 'user pr))) - (body (or (alist-get 'body pr) "No description provided.")) - (labels (mapcar (lambda (label) (alist-get 'name label)) - (alist-get 'labels pr))) - (entry-id html-url) - (title (format "#%d: %s" number title-text)) - (date (float-time (date-to-time updated))) - (content (format "

PR #%d: %s

\n

State: %s

\n

Author: %s

\n

Created: %s

\n

Updated: %s

\n%s\n

Description:

\n
%s
" - number title-text state user created updated - (if labels - (format "

Labels: %s

" - (mapconcat 'identity labels ", ")) - "") - (replace-regexp-in-string "<" "<" - (replace-regexp-in-string ">" ">" body))))) - (unless (elfeed-db-get-entry entry-id) - (let ((entry (elfeed-entry--create - :id entry-id - :title title - :link html-url - :date date - :content content - :content-type 'html - :feed-id feed-url - :tags '(unread github)))) - (push entry new-entries))))) - - ;; Call completion callback with results - (funcall completion-callback repo display-name (nreverse new-entries)))) + (new-entries ()) + (prs-to-process prs)) + ;; Set up feed metadata asynchronously + (run-with-idle-timer 0 nil + (lambda () + (let ((feed (elfeed-db-get-feed feed-url))) + (setf (elfeed-feed-url feed) feed-url + (elfeed-feed-title feed) (format "GitHub: %s [%s]" repo display-name))))) + + ;; Process PRs in chunks to avoid blocking + (cl-labels ((process-next-chunk () + (when prs-to-process + ;; Process 5 PRs at a time + (let ((chunk (cl-subseq prs-to-process 0 (min 5 (length prs-to-process))))) + (setq prs-to-process (cl-subseq prs-to-process (length chunk))) + ;; Process this chunk + (dolist (pr chunk) + (let* ((number (alist-get 'number pr)) + (title-text (alist-get 'title pr)) + (html-url (alist-get 'html_url pr)) + (state (alist-get 'state pr)) + (updated (alist-get 'updated_at pr)) + (created (alist-get 'created_at pr)) + (user (alist-get 'login (alist-get 'user pr))) + (body (or (alist-get 'body pr) "No description provided.")) + (labels (mapcar (lambda (label) (alist-get 'name label)) + (alist-get 'labels pr))) + (entry-id html-url) + (title (format "#%d: %s" number title-text)) + (date (float-time (date-to-time updated))) + (content (format "

PR #%d: %s

\n

State: %s

\n

Author: %s

\n

Created: %s

\n

Updated: %s

\n%s\n

Description:

\n
%s
" + number title-text state user created updated + (if labels + (format "

Labels: %s

" + (mapconcat 'identity labels ", ")) + "") + (replace-regexp-in-string "<" "<" + (replace-regexp-in-string ">" ">" body))))) + (unless (elfeed-db-get-entry entry-id) + (let ((entry (elfeed-entry--create + :id entry-id + :title title + :link html-url + :date date + :content content + :content-type 'html + :feed-id feed-url + :tags '(unread github)))) + (push entry new-entries))))) + ;; Schedule next chunk or call completion + (if prs-to-process + (run-with-idle-timer 0 nil #'process-next-chunk) + ;; All done - call completion callback + (funcall completion-callback repo display-name (nreverse new-entries))))))) + (process-next-chunk)))) ;;;###autoload (defun elfeed-custom-feeds-github-update () @@ -120,26 +145,30 @@ Batches all entries and updates UI once at the end to avoid blocking." ;; Completion callback that batches everything (let ((completion-callback (lambda (repo display-name new-entries) - (setq processed (1+ processed)) - (when new-entries - (setq all-new-entries (append all-new-entries new-entries))) - (push (list repo display-name (length new-entries)) results) - - ;; When all queries are done, batch add to DB and update UI once - (when (= processed total) - (let ((total-count (length all-new-entries))) - (when (> total-count 0) - (elfeed-db-add all-new-entries)) - ;; Report results for each feed - (dolist (result (nreverse results)) - (message "GitHub feed %s [%s]: %d new entries" - (nth 0 result) (nth 1 result) (nth 2 result))) - ;; Defer UI update to avoid blocking - (run-with-idle-timer 0.1 nil - (lambda () - (when (get-buffer "*elfeed-search*") - (with-current-buffer "*elfeed-search*" - (elfeed-search-update--force)))))))))) + (run-with-idle-timer 0 nil + (lambda () + (setq processed (1+ processed)) + (when new-entries + (setq all-new-entries (append all-new-entries new-entries))) + (push (list repo display-name (length new-entries)) results) + + ;; When all queries are done, batch add to DB and update UI once + (when (= processed total) + (let ((total-count (length all-new-entries))) + (when (> total-count 0) + (run-with-idle-timer 0 nil + (lambda () + (elfeed-db-add all-new-entries)))) + ;; Report results for each feed + (dolist (result (nreverse results)) + (message "GitHub feed %s [%s]: %d new entries" + (nth 0 result) (nth 1 result) (nth 2 result))) + ;; Defer UI update to avoid blocking + (run-with-idle-timer 0.2 nil + (lambda () + (when (get-buffer "*elfeed-search*") + (with-current-buffer "*elfeed-search*" + (elfeed-search-update--force))))))))))) ;; Start all async fetches (cl-loop for (repo query display-name) in elfeed-custom-github-pr-queries diff --git a/elfeed-custom-feeds-huggingface.el b/elfeed-custom-feeds-huggingface.el index 43f64c8..81f4b5b 100644 --- a/elfeed-custom-feeds-huggingface.el +++ b/elfeed-custom-feeds-huggingface.el @@ -47,22 +47,33 @@ (url-retrieve url (lambda (status) (if (plist-get status :error) - (funcall cb nil) + (run-with-idle-timer 0 nil cb nil) (goto-char (point-min)) (if (re-search-forward "\n\n" nil t) - (let* ((html (libxml-parse-html-region (point) (point-max))) - (content-div (or (dom-by-class html "model-card-content") - (dom-by-class html "prose") - (dom-by-tag html 'article)))) - (if content-div - (with-temp-buffer - (dom-print (if (listp content-div) - (car content-div) - content-div)) - (let ((html-string (buffer-string))) - (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string)))) - (funcall cb nil))) - (funcall cb nil)))) + ;; Capture raw HTML for async processing + (let ((html-text (buffer-substring-no-properties (point) (point-max)))) + ;; Process HTML parsing in idle timer to avoid blocking + (run-with-idle-timer 0 nil + (lambda () + (condition-case err + (with-temp-buffer + (insert html-text) + (let* ((html (libxml-parse-html-region (point-min) (point-max))) + (content-div (or (dom-by-class html "model-card-content") + (dom-by-class html "prose") + (dom-by-tag html 'article)))) + (if content-div + (with-temp-buffer + (dom-print (if (listp content-div) + (car content-div) + content-div)) + (let ((html-string (buffer-string))) + (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string)))) + (funcall cb nil)))) + (error + (message "Error parsing HTML for %s: %s" mid err) + (funcall cb nil)))))) + (run-with-idle-timer 0 nil cb nil)))) nil t))))) (defun elfeed-custom-feeds-huggingface--fetch-models (org-name callback) @@ -77,10 +88,21 @@ (message "Error fetching HF org %s: %s" org (plist-get status :error)) (goto-char (point-min)) (re-search-forward "\n\n") - (let* ((json-object-type 'alist) - (json-array-type 'list) - (models (json-read))) - (funcall cb org models)))) + ;; Capture JSON text for async parsing + (let ((json-text (buffer-substring-no-properties (point) (point-max)))) + ;; Parse JSON in idle timer to avoid blocking + (run-with-idle-timer 0 nil + (lambda () + (condition-case err + (with-temp-buffer + (insert json-text) + (goto-char (point-min)) + (let* ((json-object-type 'alist) + (json-array-type 'list) + (models (json-read))) + (funcall cb org models))) + (error + (message "Error parsing JSON for HF org %s: %s" org err)))))))) nil t))) (defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html) @@ -126,9 +148,11 @@ Batches all entries and updates UI once at the end to avoid blocking." (new-entries '()) (processed 0)) ;; Set up feed metadata - (let ((feed (elfeed-db-get-feed feed-url))) - (setf (elfeed-feed-url feed) feed-url - (elfeed-feed-title feed) (format "HuggingFace: %s" org-name))) + (run-with-idle-timer 0 nil + (lambda () + (let ((feed (elfeed-db-get-feed feed-url))) + (setf (elfeed-feed-url feed) feed-url + (elfeed-feed-title feed) (format "HuggingFace: %s" org-name))))) ;; Return immediately if no models (when (zerop total) @@ -144,22 +168,27 @@ Batches all entries and updates UI once at the end to avoid blocking." model-id (let ((m model)) (lambda (html) - (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html))) - (when entry - (push entry new-entries))) - (setq processed (1+ processed)) - ;; When all are processed, batch add to DB and update UI once - (when (= processed total) - (let ((count (length new-entries))) - (when (> count 0) - (elfeed-db-add (nreverse new-entries))) - (message "HuggingFace feed %s: %d new entries" org-name count) - ;; Defer UI update to avoid blocking - (run-with-idle-timer 0.1 nil - (lambda () - (when (get-buffer "*elfeed-search*") - (with-current-buffer "*elfeed-search*" - (elfeed-search-update--force))))))))) + ;; Process entry creation in idle timer to avoid blocking + (run-with-idle-timer 0 nil + (lambda () + (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html))) + (when entry + (push entry new-entries))) + (setq processed (1+ processed)) + ;; When all are processed, batch add to DB and update UI once + (when (= processed total) + (let ((count (length new-entries))) + (when (> count 0) + (run-with-idle-timer 0 nil + (lambda () + (elfeed-db-add (nreverse new-entries))))) + (message "HuggingFace feed %s: %d new entries" org-name count) + ;; Defer UI update to avoid blocking + (run-with-idle-timer 0.2 nil + (lambda () + (when (get-buffer "*elfeed-search*") + (with-current-buffer "*elfeed-search*" + (elfeed-search-update--force))))))))))) delay)) nil)) -- cgit v1.2.3