elfeed-custom-feeds-huggingface.el (11146B)
1 ;;; elfeed-custom-feeds-huggingface.el --- HuggingFace feed generator for Elfeed -*- lexical-binding: t; -*- 2 3 ;; Copyright (C) 2025 4 5 ;; SPDX-License-Identifier: GPL-3.0-or-later 6 7 ;;; Commentary: 8 9 ;; HuggingFace organization model feed generator for Elfeed. 10 11 ;;; Code: 12 13 (require 'elfeed) 14 (require 'elfeed-curl) 15 (require 'json) 16 (require 'cl-lib) 17 (require 'dom) 18 19 (defgroup elfeed-custom-feeds-huggingface nil 20 "HuggingFace feed generator for Elfeed." 21 :group 'elfeed 22 :prefix "elfeed-custom-huggingface-") 23 24 (defcustom elfeed-custom-huggingface-orgs 25 '("sentence-transformers" "google-bert") 26 "List of HuggingFace organizations to monitor." 27 :type '(repeat string) 28 :group 'elfeed-custom-feeds-huggingface) 29 30 (defcustom elfeed-custom-huggingface-request-delay 0.5 31 "Delay in seconds between HuggingFace requests to avoid rate limiting." 32 :type 'number 33 :group 'elfeed-custom-feeds-huggingface) 34 35 (defun elfeed-custom-feeds-huggingface--strip-svg-tags (html) 36 "Remove SVG tags from HTML string." 37 (when html 38 (replace-regexp-in-string "<svg[^>]*>.*?</svg>" "" html nil nil))) 39 40 (defun elfeed-custom-feeds-huggingface--fetch-model-card (model-id callback delay) 41 "Fetch model card HTML for MODEL-ID from HuggingFace after DELAY seconds." 42 (let ((url (format "https://huggingface.co/%s" model-id)) 43 (cb callback) 44 (mid model-id)) 45 (run-at-time delay nil 46 (lambda () 47 (elfeed-curl-enqueue url 48 (lambda (status) 49 (if (not status) 50 (run-with-idle-timer 0 nil cb nil) 51 ;; Capture raw HTML for async processing 52 (let ((html-text (buffer-substring-no-properties (point-min) (point-max)))) 53 ;; Process HTML parsing in idle timer to avoid blocking 54 (run-with-idle-timer 0 nil 55 (lambda () 56 (condition-case err 57 (with-temp-buffer 58 (insert html-text) 59 (let* ((html (libxml-parse-html-region (point-min) (point-max))) 60 (content-div (or (dom-by-class html "model-card-content") 61 (dom-by-class html "prose") 62 (dom-by-tag html 'article)))) 63 (if content-div 64 (with-temp-buffer 65 (dom-print (if (listp content-div) 66 (car content-div) 67 content-div)) 68 (let ((html-string (buffer-string))) 69 (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string)))) 70 (funcall cb nil)))) 71 (error 72 (message "Error parsing HTML for %s: %s" mid err) 73 (funcall cb nil))))))))))))) 74 75 (defun elfeed-custom-feeds-huggingface--fetch-models (org-name callback) 76 "Fetch models for ORG-NAME from HuggingFace API and call CALLBACK with results." 77 (let ((url (format "https://huggingface.co/api/models?author=%s&sort=lastModified&direction=-1&limit=20" 78 (url-hexify-string org-name))) 79 (cb callback) 80 (org org-name)) 81 (elfeed-curl-enqueue url 82 (lambda (status) 83 (if (not status) 84 (message "Error fetching HF org %s: %s" org elfeed-curl-error-message) 85 ;; Capture JSON text for async parsing 86 (let ((json-text (buffer-substring-no-properties (point-min) (point-max)))) 87 ;; Parse JSON in idle timer to avoid blocking 88 (run-with-idle-timer 0 nil 89 (lambda () 90 (condition-case err 91 (with-temp-buffer 92 (insert json-text) 93 (goto-char (point-min)) 94 (let* ((json-object-type 'alist) 95 (json-array-type 'list) 96 (models (json-read))) 97 (funcall cb org models))) 98 (error 99 (message "Error parsing JSON for HF org %s: %s" org err))))))))))) 100 101 (defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html) 102 "Create an elfeed entry for MODEL with model card HTML content. 103 Returns the entry if it's new, nil otherwise." 104 (let* ((feed-url (format "https://huggingface.co/%s" org-name)) 105 (model-id (alist-get 'id model)) 106 (last-modified (alist-get 'lastModified model)) 107 (downloads (or (alist-get 'downloads model) 0)) 108 (pipeline (or (alist-get 'pipeline_tag model) "model")) 109 (tags (alist-get 'tags model)) 110 (entry-id (format "%s:%s" feed-url model-id)) 111 (link (format "https://huggingface.co/%s" model-id)) 112 (title (format "[%s] %s" org-name model-id)) 113 (date (float-time (date-to-time last-modified))) 114 (metadata (format "<h2>%s</h2>\n<p><strong>Type:</strong> %s</p>\n<p><strong>Downloads:</strong> %d</p>\n<p><strong>Last Modified:</strong> %s</p>\n%s\n<hr>\n" 115 model-id 116 pipeline 117 downloads 118 last-modified 119 (if tags 120 (format "<p><strong>Tags:</strong> %s</p>" 121 (mapconcat 'identity (cl-subseq tags 0 (min 10 (length tags))) ", ")) 122 ""))) 123 (content (concat metadata 124 (or model-card-html "<p><em>Model card not available</em></p>")))) 125 (unless (elfeed-db-get-entry entry-id) 126 (elfeed-entry--create 127 :id entry-id 128 :title title 129 :link link 130 :date date 131 :content content 132 :content-type 'html 133 :feed-id feed-url 134 :tags '(unread huggingface))))) 135 136 (defun elfeed-custom-feeds-huggingface--models-to-elfeed (org-name models) 137 "Convert HuggingFace MODELS to elfeed entries for ORG-NAME. 138 Batches all entries and updates UI once at the end to avoid blocking." 139 (let* ((feed-url (format "https://huggingface.co/%s" org-name)) 140 (total (length models)) 141 (new-entries '()) 142 (processed 0)) 143 ;; Set up feed metadata 144 (run-with-idle-timer 0 nil 145 (lambda () 146 (let ((feed (elfeed-db-get-feed feed-url))) 147 (setf (elfeed-feed-url feed) feed-url 148 (elfeed-feed-title feed) (format "HuggingFace: %s" org-name))))) 149 150 (if (zerop total) 151 (message "HuggingFace feed %s: no models to process" org-name) 152 ;; Fetch model cards asynchronously and batch entries 153 (cl-loop for model in models 154 for i from 0 155 for model-id = (alist-get 'id model) 156 for delay = (* i elfeed-custom-huggingface-request-delay) 157 do (elfeed-custom-feeds-huggingface--fetch-model-card 158 model-id 159 (let ((m model)) 160 (lambda (html) 161 ;; Process entry creation in idle timer to avoid blocking 162 (run-with-idle-timer 0 nil 163 (lambda () 164 (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html))) 165 (when entry 166 (push entry new-entries))) 167 (setq processed (1+ processed)) 168 ;; When all are processed, batch add to DB and update UI once 169 (when (= processed total) 170 (let ((count (length new-entries))) 171 (when (> count 0) 172 (run-with-idle-timer 0 nil 173 (lambda () 174 (elfeed-db-add (nreverse new-entries))))) 175 (message "HuggingFace feed %s: %d new entries" org-name count) 176 ;; Defer UI update to avoid blocking 177 (run-with-idle-timer 0.2 nil 178 (lambda () 179 (when (get-buffer "*elfeed-search*") 180 (with-current-buffer "*elfeed-search*" 181 (elfeed-search-update--force))))))))))) 182 delay)) 183 nil))) 184 185 ;;;###autoload 186 (defun elfeed-custom-feeds-huggingface-update () 187 "Update all HuggingFace organization feeds asynchronously. 188 Returns immediately while updates happen in the background." 189 (interactive) 190 (dolist (org elfeed-custom-huggingface-orgs) 191 (elfeed-custom-feeds-huggingface--fetch-models org #'elfeed-custom-feeds-huggingface--models-to-elfeed)) 192 nil) 193 194 (provide 'elfeed-custom-feeds-huggingface) 195 196 ;;; elfeed-custom-feeds-huggingface.el ends here