;;; elfeed-custom-feeds-huggingface.el --- HuggingFace feed generator for Elfeed -*- lexical-binding: t; -*- ;; Copyright (C) 2025 ;; SPDX-License-Identifier: GPL-3.0-or-later ;;; Commentary: ;; HuggingFace organization model feed generator for Elfeed. ;;; Code: (require 'elfeed) (require 'elfeed-curl) (require 'json) (require 'cl-lib) (require 'dom) (defgroup elfeed-custom-feeds-huggingface nil "HuggingFace feed generator for Elfeed." :group 'elfeed :prefix "elfeed-custom-huggingface-") (defcustom elfeed-custom-huggingface-orgs '("sentence-transformers" "google-bert") "List of HuggingFace organizations to monitor." :type '(repeat string) :group 'elfeed-custom-feeds-huggingface) (defcustom elfeed-custom-huggingface-request-delay 0.5 "Delay in seconds between HuggingFace requests to avoid rate limiting." :type 'number :group 'elfeed-custom-feeds-huggingface) (defun elfeed-custom-feeds-huggingface--strip-svg-tags (html) "Remove SVG tags from HTML string." (when html (replace-regexp-in-string "]*>.*?" "" html nil nil))) (defun elfeed-custom-feeds-huggingface--fetch-model-card (model-id callback delay) "Fetch model card HTML for MODEL-ID from HuggingFace after DELAY seconds." (let ((url (format "https://huggingface.co/%s" model-id)) (cb callback) (mid model-id)) (run-at-time delay nil (lambda () (elfeed-curl-enqueue url (lambda (status) (if (not status) (run-with-idle-timer 0 nil cb nil) ;; Capture raw HTML for async processing (let ((html-text (buffer-substring-no-properties (point-min) (point-max)))) ;; Process HTML parsing in idle timer to avoid blocking (run-with-idle-timer 0 nil (lambda () (condition-case err (with-temp-buffer (insert html-text) (let* ((html (libxml-parse-html-region (point-min) (point-max))) (content-div (or (dom-by-class html "model-card-content") (dom-by-class html "prose") (dom-by-tag html 'article)))) (if content-div (with-temp-buffer (dom-print (if (listp content-div) (car content-div) content-div)) (let ((html-string (buffer-string))) (funcall cb (elfeed-custom-feeds-huggingface--strip-svg-tags html-string)))) (funcall cb nil)))) (error (message "Error parsing HTML for %s: %s" mid err) (funcall cb nil))))))))))))) (defun elfeed-custom-feeds-huggingface--fetch-models (org-name callback) "Fetch models for ORG-NAME from HuggingFace API and call CALLBACK with results." (let ((url (format "https://huggingface.co/api/models?author=%s&sort=lastModified&direction=-1&limit=20" (url-hexify-string org-name))) (cb callback) (org org-name)) (elfeed-curl-enqueue url (lambda (status) (if (not status) (message "Error fetching HF org %s: %s" org elfeed-curl-error-message) ;; Capture JSON text for async parsing (let ((json-text (buffer-substring-no-properties (point-min) (point-max)))) ;; Parse JSON in idle timer to avoid blocking (run-with-idle-timer 0 nil (lambda () (condition-case err (with-temp-buffer (insert json-text) (goto-char (point-min)) (let* ((json-object-type 'alist) (json-array-type 'list) (models (json-read))) (funcall cb org models))) (error (message "Error parsing JSON for HF org %s: %s" org err))))))))))) (defun elfeed-custom-feeds-huggingface--create-entry (org-name model model-card-html) "Create an elfeed entry for MODEL with model card HTML content. Returns the entry if it's new, nil otherwise." (let* ((feed-url (format "https://huggingface.co/%s" org-name)) (model-id (alist-get 'id model)) (last-modified (alist-get 'lastModified model)) (downloads (or (alist-get 'downloads model) 0)) (pipeline (or (alist-get 'pipeline_tag model) "model")) (tags (alist-get 'tags model)) (entry-id (format "%s:%s" feed-url model-id)) (link (format "https://huggingface.co/%s" model-id)) (title (format "[%s] %s" org-name model-id)) (date (float-time (date-to-time last-modified))) (metadata (format "

%s

\n

Type: %s

\n

Downloads: %d

\n

Last Modified: %s

\n%s\n
\n" model-id pipeline downloads last-modified (if tags (format "

Tags: %s

" (mapconcat 'identity (cl-subseq tags 0 (min 10 (length tags))) ", ")) ""))) (content (concat metadata (or model-card-html "

Model card not available

")))) (unless (elfeed-db-get-entry entry-id) (elfeed-entry--create :id entry-id :title title :link link :date date :content content :content-type 'html :feed-id feed-url :tags '(unread huggingface))))) (defun elfeed-custom-feeds-huggingface--models-to-elfeed (org-name models) "Convert HuggingFace MODELS to elfeed entries for ORG-NAME. Batches all entries and updates UI once at the end to avoid blocking." (let* ((feed-url (format "https://huggingface.co/%s" org-name)) (total (length models)) (new-entries '()) (processed 0)) ;; Set up feed metadata (run-with-idle-timer 0 nil (lambda () (let ((feed (elfeed-db-get-feed feed-url))) (setf (elfeed-feed-url feed) feed-url (elfeed-feed-title feed) (format "HuggingFace: %s" org-name))))) (if (zerop total) (message "HuggingFace feed %s: no models to process" org-name) ;; Fetch model cards asynchronously and batch entries (cl-loop for model in models for i from 0 for model-id = (alist-get 'id model) for delay = (* i elfeed-custom-huggingface-request-delay) do (elfeed-custom-feeds-huggingface--fetch-model-card model-id (let ((m model)) (lambda (html) ;; Process entry creation in idle timer to avoid blocking (run-with-idle-timer 0 nil (lambda () (let ((entry (elfeed-custom-feeds-huggingface--create-entry org-name m html))) (when entry (push entry new-entries))) (setq processed (1+ processed)) ;; When all are processed, batch add to DB and update UI once (when (= processed total) (let ((count (length new-entries))) (when (> count 0) (run-with-idle-timer 0 nil (lambda () (elfeed-db-add (nreverse new-entries))))) (message "HuggingFace feed %s: %d new entries" org-name count) ;; Defer UI update to avoid blocking (run-with-idle-timer 0.2 nil (lambda () (when (get-buffer "*elfeed-search*") (with-current-buffer "*elfeed-search*" (elfeed-search-update--force))))))))))) delay)) nil))) ;;;###autoload (defun elfeed-custom-feeds-huggingface-update () "Update all HuggingFace organization feeds asynchronously. Returns immediately while updates happen in the background." (interactive) (dolist (org elfeed-custom-huggingface-orgs) (elfeed-custom-feeds-huggingface--fetch-models org #'elfeed-custom-feeds-huggingface--models-to-elfeed)) nil) (provide 'elfeed-custom-feeds-huggingface) ;;; elfeed-custom-feeds-huggingface.el ends here