blob: 703b92e53c9c7d72b9e204930a6a44f770bb90a5 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
|
;;; elfeed-custom-feeds-huggingface.el --- HuggingFace feed generator for Elfeed -*- lexical-binding: t; -*-
;; Copyright (C) 2025
;; SPDX-License-Identifier: GPL-3.0-or-later
;;; Commentary:
;; HuggingFace organization model feed generator for Elfeed.
;;; Code:
(require 'elfeed)
(require 'elfeed-curl)
(require 'json)
(require 'cl-lib)
(require 'dom)
(defgroup elfeed-custom-feeds-huggingface nil
"HuggingFace feed generator for Elfeed."
:group 'elfeed
:prefix "elfeed-custom-huggingface-")
(defcustom elfeed-custom-huggingface-orgs
'("sentence-transformers" "google-bert")
"List of HuggingFace organizations to monitor."
:type '(repeat string)
:group 'elfeed-custom-feeds-huggingface)
(defcustom elfeed-custom-huggingface-request-delay 0.5
"Delay in seconds between HuggingFace requests to avoid rate limiting."
:type 'number
:group 'elfeed-custom-feeds-huggingface)
(defun elfeed-custom-feeds-huggingface--strip-svg-tags (html)
"Remove SVG tags from HTML string."
(when html
(replace-regexp-in-string "<svg[^>]*>.*?</svg>" "" html nil nil)))
(defun elfeed-custom-feeds-huggingface--fetch-model-card (model-id callback delay)
"Fetch model card HTML for MODEL-ID from HuggingFace after DELAY seconds."
(let ((url (format "https://huggingface.co/%s" model-id))
(cb callback)
(mid model-id))
(run-at-time delay nil
(lambda ()
(elfeed-curl-enqueue url
(lambda (status)
(if (not status)
(run-with-idle-timer 0 nil cb nil)
;; Capture raw HTML for async processing
(let ((html-text (buffer-substring-no-properties (point-min) (point-max))))
;; Process HTML parsing in idle timer to avoid blocking
(run-with-idle-timer 0 nil
(lambda ()
(condition-case err
(with-temp-buffer
(insert html-text)
(let* ((html (libxml-parse-html-region (point-min) (point-max)))
(content-div (or (dom-by-class html "model-card-content")
(dom-by-class html "prose")
(dom-by-tag html 'article))))
(if content-div
(with-temp-buffer
(dom-print (if (listp content-div)
(car content-div)
content-div))
(let ((html-string (buffer-string)))
(funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string))))
(funcall cb nil))))
(error
(message "Error parsing HTML for %s: %s" mid err)
(funcall cb nil)))))))))))))
(defun elfeed-custom-feeds-huggingface--fetch-models (org-name callback)
"Fetch models for ORG-NAME from HuggingFace API and call CALLBACK with results."
(let ((url (format "https://huggingface.co/api/models?author=%s&sort=lastModified&direction=-1&limit=20"
(url-hexify-string org-name)))
(cb callback)
(org org-name))
(elfeed-curl-enqueue url
(lambda (status)
(if (not status)
(message "Error fetching HF org %s: %s" org elfeed-curl-error-message)
;; Capture JSON text for async parsing
(let ((json-text (buffer-substring-no-properties (point-min) (point-max))))
;; Parse JSON in idle timer to avoid blocking
(run-with-idle-timer 0 nil
(lambda ()
(condition-case err
(with-temp-buffer
(insert json-text)
(goto-char (point-min))
(let* ((json-object-type 'alist)
(json-array-type 'list)
(models (json-read)))
(funcall cb org models)))
(error
(message "Error parsing JSON for HF org %s: %s" org err)))))))))))
(defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html)
"Create an elfeed entry for MODEL with model card HTML content.
Returns the entry if it's new, nil otherwise."
(let* ((feed-url (format "https://huggingface.co/%s" org-name))
(model-id (alist-get 'id model))
(last-modified (alist-get 'lastModified model))
(downloads (or (alist-get 'downloads model) 0))
(pipeline (or (alist-get 'pipeline_tag model) "model"))
(tags (alist-get 'tags model))
(entry-id (format "%s:%s" feed-url model-id))
(link (format "https://huggingface.co/%s" model-id))
(title (format "[%s] %s" org-name model-id))
(date (float-time (date-to-time last-modified)))
(metadata (format "<h2>%s</h2>\n<p><strong>Type:</strong> %s</p>\n<p><strong>Downloads:</strong> %d</p>\n<p><strong>Last Modified:</strong> %s</p>\n%s\n<hr>\n"
model-id
pipeline
downloads
last-modified
(if tags
(format "<p><strong>Tags:</strong> %s</p>"
(mapconcat 'identity (cl-subseq tags 0 (min 10 (length tags))) ", "))
"")))
(content (concat metadata
(or model-card-html "<p><em>Model card not available</em></p>"))))
(unless (elfeed-db-get-entry entry-id)
(elfeed-entry--create
:id entry-id
:title title
:link link
:date date
:content content
:content-type 'html
:feed-id feed-url
:tags '(unread huggingface)))))
(defun elfeed-custom-feeds-huggingface--models-to-elfeed (org-name models)
"Convert HuggingFace MODELS to elfeed entries for ORG-NAME.
Batches all entries and updates UI once at the end to avoid blocking."
(let* ((feed-url (format "https://huggingface.co/%s" org-name))
(total (length models))
(new-entries '())
(processed 0))
;; Set up feed metadata
(run-with-idle-timer 0 nil
(lambda ()
(let ((feed (elfeed-db-get-feed feed-url)))
(setf (elfeed-feed-url feed) feed-url
(elfeed-feed-title feed) (format "HuggingFace: %s" org-name)))))
(if (zerop total)
(message "HuggingFace feed %s: no models to process" org-name)
;; Fetch model cards asynchronously and batch entries
(cl-loop for model in models
for i from 0
for model-id = (alist-get 'id model)
for delay = (* i elfeed-custom-huggingface-request-delay)
do (elfeed-custom-feeds-huggingface--fetch-model-card
model-id
(let ((m model))
(lambda (html)
;; Process entry creation in idle timer to avoid blocking
(run-with-idle-timer 0 nil
(lambda ()
(let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html)))
(when entry
(push entry new-entries)))
(setq processed (1+ processed))
;; When all are processed, batch add to DB and update UI once
(when (= processed total)
(let ((count (length new-entries)))
(when (> count 0)
(run-with-idle-timer 0 nil
(lambda ()
(elfeed-db-add (nreverse new-entries)))))
(message "HuggingFace feed %s: %d new entries" org-name count)
;; Defer UI update to avoid blocking
(run-with-idle-timer 0.2 nil
(lambda ()
(when (get-buffer "*elfeed-search*")
(with-current-buffer "*elfeed-search*"
(elfeed-search-update--force)))))))))))
delay))
nil)))
;;;###autoload
(defun elfeed-custom-feeds-huggingface-update ()
"Update all HuggingFace organization feeds asynchronously.
Returns immediately while updates happen in the background."
(interactive)
(dolist (org elfeed-custom-huggingface-orgs)
(elfeed-custom-feeds-huggingface--fetch-models org #'elfeed-custom-feeds-huggingface--models-to-elfeed))
nil)
(provide 'elfeed-custom-feeds-huggingface)
;;; elfeed-custom-feeds-huggingface.el ends here
|
