diff options
Diffstat (limited to 'elfeed-custom-feeds-huggingface.el')
| -rw-r--r-- | elfeed-custom-feeds-huggingface.el | 114 |
1 files changed, 52 insertions, 62 deletions
diff --git a/elfeed-custom-feeds-huggingface.el b/elfeed-custom-feeds-huggingface.el index 81f4b5b..703b92e 100644 --- a/elfeed-custom-feeds-huggingface.el +++ b/elfeed-custom-feeds-huggingface.el | |||
| @@ -11,7 +11,7 @@ | |||
| 11 | ;;; Code: | 11 | ;;; Code: |
| 12 | 12 | ||
| 13 | (require 'elfeed) | 13 | (require 'elfeed) |
| 14 | (require 'url) | 14 | (require 'elfeed-curl) |
| 15 | (require 'json) | 15 | (require 'json) |
| 16 | (require 'cl-lib) | 16 | (require 'cl-lib) |
| 17 | (require 'dom) | 17 | (require 'dom) |
| @@ -44,37 +44,33 @@ | |||
| 44 | (mid model-id)) | 44 | (mid model-id)) |
| 45 | (run-at-time delay nil | 45 | (run-at-time delay nil |
| 46 | (lambda () | 46 | (lambda () |
| 47 | (url-retrieve url | 47 | (elfeed-curl-enqueue url |
| 48 | (lambda (status) | 48 | (lambda (status) |
| 49 | (if (plist-get status :error) | 49 | (if (not status) |
| 50 | (run-with-idle-timer 0 nil cb nil) | 50 | (run-with-idle-timer 0 nil cb nil) |
| 51 | (goto-char (point-min)) | 51 | ;; Capture raw HTML for async processing |
| 52 | (if (re-search-forward "\n\n" nil t) | 52 | (let ((html-text (buffer-substring-no-properties (point-min) (point-max)))) |
| 53 | ;; Capture raw HTML for async processing | 53 | ;; Process HTML parsing in idle timer to avoid blocking |
| 54 | (let ((html-text (buffer-substring-no-properties (point) (point-max)))) | 54 | (run-with-idle-timer 0 nil |
| 55 | ;; Process HTML parsing in idle timer to avoid blocking | 55 | (lambda () |
| 56 | (run-with-idle-timer 0 nil | 56 | (condition-case err |
| 57 | (lambda () | 57 | (with-temp-buffer |
| 58 | (condition-case err | 58 | (insert html-text) |
| 59 | (with-temp-buffer | 59 | (let* ((html (libxml-parse-html-region (point-min) (point-max))) |
| 60 | (insert html-text) | 60 | (content-div (or (dom-by-class html "model-card-content") |
| 61 | (let* ((html (libxml-parse-html-region (point-min) (point-max))) | 61 | (dom-by-class html "prose") |
| 62 | (content-div (or (dom-by-class html "model-card-content") | 62 | (dom-by-tag html 'article)))) |
| 63 | (dom-by-class html "prose") | 63 | (if content-div |
| 64 | (dom-by-tag html 'article)))) | 64 | (with-temp-buffer |
| 65 | (if content-div | 65 | (dom-print (if (listp content-div) |
| 66 | (with-temp-buffer | 66 | (car content-div) |
| 67 | (dom-print (if (listp content-div) | 67 | content-div)) |
| 68 | (car content-div) | 68 | (let ((html-string (buffer-string))) |
| 69 | content-div)) | 69 | (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string)))) |
| 70 | (let ((html-string (buffer-string))) | 70 | (funcall cb nil)))) |
| 71 | (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string)))) | 71 | (error |
| 72 | (funcall cb nil)))) | 72 | (message "Error parsing HTML for %s: %s" mid err) |
| 73 | (error | 73 | (funcall cb nil))))))))))))) |
| 74 | (message "Error parsing HTML for %s: %s" mid err) | ||
| 75 | (funcall cb nil)))))) | ||
| 76 | (run-with-idle-timer 0 nil cb nil)))) | ||
| 77 | nil t))))) | ||
| 78 | 74 | ||
| 79 | (defun elfeed-custom-feeds-huggingface--fetch-models (org-name callback) | 75 | (defun elfeed-custom-feeds-huggingface--fetch-models (org-name callback) |
| 80 | "Fetch models for ORG-NAME from HuggingFace API and call CALLBACK with results." | 76 | "Fetch models for ORG-NAME from HuggingFace API and call CALLBACK with results." |
| @@ -82,28 +78,25 @@ | |||
| 82 | (url-hexify-string org-name))) | 78 | (url-hexify-string org-name))) |
| 83 | (cb callback) | 79 | (cb callback) |
| 84 | (org org-name)) | 80 | (org org-name)) |
| 85 | (url-retrieve url | 81 | (elfeed-curl-enqueue url |
| 86 | (lambda (status) | 82 | (lambda (status) |
| 87 | (if (plist-get status :error) | 83 | (if (not status) |
| 88 | (message "Error fetching HF org %s: %s" org (plist-get status :error)) | 84 | (message "Error fetching HF org %s: %s" org elfeed-curl-error-message) |
| 89 | (goto-char (point-min)) | 85 | ;; Capture JSON text for async parsing |
| 90 | (re-search-forward "\n\n") | 86 | (let ((json-text (buffer-substring-no-properties (point-min) (point-max)))) |
| 91 | ;; Capture JSON text for async parsing | 87 | ;; Parse JSON in idle timer to avoid blocking |
| 92 | (let ((json-text (buffer-substring-no-properties (point) (point-max)))) | 88 | (run-with-idle-timer 0 nil |
| 93 | ;; Parse JSON in idle timer to avoid blocking | 89 | (lambda () |
| 94 | (run-with-idle-timer 0 nil | 90 | (condition-case err |
| 95 | (lambda () | 91 | (with-temp-buffer |
| 96 | (condition-case err | 92 | (insert json-text) |
| 97 | (with-temp-buffer | 93 | (goto-char (point-min)) |
| 98 | (insert json-text) | 94 | (let* ((json-object-type 'alist) |
| 99 | (goto-char (point-min)) | 95 | (json-array-type 'list) |
| 100 | (let* ((json-object-type 'alist) | 96 | (models (json-read))) |
| 101 | (json-array-type 'list) | 97 | (funcall cb org models))) |
| 102 | (models (json-read))) | 98 | (error |
| 103 | (funcall cb org models))) | 99 | (message "Error parsing JSON for HF org %s: %s" org err))))))))))) |
| 104 | (error | ||
| 105 | (message "Error parsing JSON for HF org %s: %s" org err)))))))) | ||
| 106 | nil t))) | ||
| 107 | 100 | ||
| 108 | (defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html) | 101 | (defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html) |
| 109 | "Create an elfeed entry for MODEL with model card HTML content. | 102 | "Create an elfeed entry for MODEL with model card HTML content. |
| @@ -154,13 +147,10 @@ Batches all entries and updates UI once at the end to avoid blocking." | |||
| 154 | (setf (elfeed-feed-url feed) feed-url | 147 | (setf (elfeed-feed-url feed) feed-url |
| 155 | (elfeed-feed-title feed) (format "HuggingFace: %s" org-name))))) | 148 | (elfeed-feed-title feed) (format "HuggingFace: %s" org-name))))) |
| 156 | 149 | ||
| 157 | ;; Return immediately if no models | 150 | (if (zerop total) |
| 158 | (when (zerop total) | 151 | (message "HuggingFace feed %s: no models to process" org-name) |
| 159 | (message "HuggingFace feed %s: no models to process" org-name) | 152 | ;; Fetch model cards asynchronously and batch entries |
| 160 | (cl-return-from elfeed-custom-feeds-huggingface--models-to-elfeed nil)) | 153 | (cl-loop for model in models |
| 161 | |||
| 162 | ;; Fetch model cards asynchronously and batch entries | ||
| 163 | (cl-loop for model in models | ||
| 164 | for i from 0 | 154 | for i from 0 |
| 165 | for model-id = (alist-get 'id model) | 155 | for model-id = (alist-get 'id model) |
| 166 | for delay = (* i elfeed-custom-huggingface-request-delay) | 156 | for delay = (* i elfeed-custom-huggingface-request-delay) |
| @@ -190,7 +180,7 @@ Batches all entries and updates UI once at the end to avoid blocking." | |||
| 190 | (with-current-buffer "*elfeed-search*" | 180 | (with-current-buffer "*elfeed-search*" |
| 191 | (elfeed-search-update--force))))))))))) | 181 | (elfeed-search-update--force))))))))))) |
| 192 | delay)) | 182 | delay)) |
| 193 | nil)) | 183 | nil))) |
| 194 | 184 | ||
| 195 | ;;;###autoload | 185 | ;;;###autoload |
| 196 | (defun elfeed-custom-feeds-huggingface-update () | 186 | (defun elfeed-custom-feeds-huggingface-update () |