commit 133712b6981c79c89f0251e4f131e9049a52f1db
parent 2d76af7f779fdf79a9a87bbc58b86789d79062a7
Author: Vineet Kumar <git@vineetk.net>
Date: Fri, 20 Nov 2026 02:15:27 +0100
rewrite by reusing eww functions
so now only network requests it makes is to `pulpie-server'.
and its dom stuff is much simpler too.
Diffstat:
| M | pulpie.el | | | 231 | ++++++++++++++++++++++++++++--------------------------------------------------- |
1 file changed, 80 insertions(+), 151 deletions(-)
diff --git a/pulpie.el b/pulpie.el
@@ -5,32 +5,30 @@
;;; Commentary:
-;; One entry point:
-;;
-;; (pulpie-main-html "https://example.com/article")
-;; => "<main><h1>...</h1><p>...</p></main>"
-;;
-;; It returns the page's main content as an HTML fragment - navigation,
-;; sidebars, comments and footers dropped - intended as the reader step in
-;; front of eww, which otherwise shows the whole document.
-;;
-;; Classification runs on a llama-server serving a pulpie GGUF:
+;; Enable with (pulpie-eww-mode 1): wraps `eww-readable-dom' with
+;; `pulpie-readable-dom', so eww's reader classifies main content with a
+;; pulpie classifier instead of heuristics. The DOM eww already parsed is
+;; used as-is; the only requests pulpie makes go to `pulpie-server', a
+;; llama-server (or compatible) exposing the model, e.g.:
;;
;; llama-server -m pulpie-orange-small-cls-f16.gguf --embeddings \
-;; -b 8192 -ub 8192 -c 8192 --port 8888
+;; -b 8192 -ub 8192 -c 8192
;;
-;; How it works: the page is split into leaf blocks, packed into the model's
-;; chunk template
+;; The DOM is split into leaf blocks, packed into the model's chunk template
;;
;; [BOS] block0 <|sep|> block1 <|sep|> ... <|sep|> [EOS]
;;
-;; and every <|sep|> row of /embedding carries that block's [other, main]
-;; logits. Token ids come from the server's /tokenize endpoint, so they match
-;; the tokenizer baked into the GGUF exactly.
+;; each <|sep|> row of /embedding carrying that block's [other, main]
+;; logits. Token ids come from /tokenize so they match the model's own
+;; tokenizer.
;;
;; Blocks are classified as leaves but emitted as whole subtrees: each kept
;; block is lifted to the largest ancestor containing only kept blocks, so
;; <ul>, <table> and section wrappers survive.
+;;
+;; In terms of performance overhead, it's around 0.1-0.8s when
+;; enabling reader mode on my machines. That's with the classifier
+;; running on CPU and not GPU.
;;; Code:
@@ -39,21 +37,17 @@
(require 'json)
(require 'pcase)
(require 'url)
-(require 'url-http)
(defgroup pulpie nil
- "Main-content extraction with a pulpie GGUF behind llama-server."
+ "Main-content extraction with a pulpie classifier."
:group 'hypermedia)
-;; (defcustom pulpie-server "http://localhost:8888"
-;; "Base URL of the llama-server serving a pulpie GGUF."
-;; :type 'string :group 'pulpie)
(defcustom pulpie-server "http://llm.demiurge.epistemia/upstream/pulpie-orange-small"
- "Base URL of the llama-server serving a pulpie GGUF."
+ "Base URL of the llama-server exposing the pulpie classifier."
:type 'string :group 'pulpie)
(defcustom pulpie-max-tokens 8192
- "Token budget per request; must not exceed the server's -b/-ub/-c value."
+ "Token budget per request; must not exceed the server's context size."
:type 'integer :group 'pulpie)
(defconst pulpie-bos "<|begin_of_text|>" "Chunk prefix.")
@@ -65,15 +59,6 @@
'(p li h1 h2 h3 h4 h5 h6 pre blockquote figcaption td th dd dt summary)
"Block-level tags whose text is classified (leaf elements only).")
-(defconst pulpie-noise-tags
- '(script style noscript iframe template svg canvas form)
- "Tags dropped when merged blocks are serialized.")
-
-(defconst pulpie-void-tags
- '("area" "base" "br" "col" "embed" "hr" "img" "input" "link" "meta"
- "source" "track" "wbr")
- "HTML elements that must not be printed with a closing tag.")
-
;;; HTTP
(defun pulpie--header-end ()
@@ -83,55 +68,30 @@
(if (re-search-forward "\r?\n\r?\n" nil t) (match-end 0) (point-min))))
(defun pulpie--body (buffer url)
- "Return BUFFER's decoded body as a string, then kill BUFFER.
+ "Return BUFFER's body decoded as UTF-8, then kill BUFFER.
URL is only used in the error message. Signals on a 4xx/5xx status."
(with-current-buffer buffer
(unwind-protect
- (let* ((status (save-excursion
- (goto-char (point-min))
- (when (looking-at "HTTP/[0-9.]+ \\([0-9]+\\)")
- (string-to-number (match-string 1)))))
- (header-end (pulpie--header-end))
- (charset (save-excursion
- (goto-char (point-min))
- (when (re-search-forward
- "^Content-Type:.*?charset=\\([^; \r\n]+\\)"
- header-end t)
- (match-string 1))))
- ;; no charset is UTF-8, not the ISO-8859-1 of an old RFC
- (coding (let ((sym (and charset (ignore-errors (intern charset)))))
- (if (and sym (coding-system-p sym)) sym 'utf-8))))
- (when (and status (>= status 400))
- (error "Request to %s failed (%s): %s"
- url status
+ (let ((header-end (pulpie--header-end)))
+ (goto-char (point-min))
+ (when (and (looking-at "HTTP/[0-9.]+ \([0-9]+\)")
+ (>= (string-to-number (match-string 1)) 400))
+ (error "Request to %s failed (%s): %s" url (match-string 1)
(string-trim
(buffer-substring-no-properties header-end (point-max)))))
- (let ((body (buffer-substring-no-properties (pulpie--header-end) (point-max))))
- ;; url.el leaves the body as raw bytes (a unibyte string); decode
- ;; them here. decode-coding-region on the buffer is a no-op.
+ (let ((body (buffer-substring-no-properties header-end (point-max))))
+ ;; url.el may leave the body as raw bytes
(if (multibyte-string-p body)
body
- (decode-coding-string body coding))))
+ (decode-coding-string body 'utf-8))))
(kill-buffer buffer))))
-(defun pulpie--get (url)
- "Fetch URL and return its decoded body."
- (let ((url-user-agent url-user-agent)
- (url-mime-accept-string "text/html,application/xhtml+xml,*/*")
- (url-show-status nil)
- (url-request-method "GET")
- (url-request-extra-headers
- (list (cons "User-Agent" pulpie-user-agent)
- (cons "Accept" "text/html,application/xhtml+xml,*/*"))))
- (pulpie--body (url-retrieve-synchronously url t nil 60) url)))
-
(defun pulpie--post-json (url payload)
"POST PAYLOAD as JSON to URL and return the parsed response."
(let* ((url-show-status nil)
(url-request-method "POST")
(url-request-extra-headers
- (list (cons "Content-Type" "application/json")
- (cons "User-Agent" pulpie-user-agent)))
+ (list (cons "Content-Type" "application/json")))
(url-request-data (encode-coding-string (json-encode payload) 'utf-8)))
(json-parse-string (pulpie--body (url-retrieve-synchronously url t nil 300) url)
;; keys come back interned, hence `assq' below
@@ -180,30 +140,22 @@ URL is only used in the error message. Signals on a 4xx/5xx status."
(defun pulpie--escape (s)
"Escape S for the classifier markup.
-Mirrors Python's html.escape(s, quote=True), which is what the model was
-verified against - including the apostrophe, because an apostrophe becomes
-' and that changes the token ids the classifier sees."
+The model was trained on markup where the apostrophe becomes ', so
+this set of escapes, and no other, must be preserved: different escaping
+yields different token ids."
(let ((s (replace-regexp-in-string "&" "&" s)))
(setq s (replace-regexp-in-string "<" "<" s))
(setq s (replace-regexp-in-string ">" ">" s))
(setq s (replace-regexp-in-string "\"" """ s))
(replace-regexp-in-string "'" "'" s)))
-(defun pulpie--collapse (s)
- "Collapse every whitespace run in S to a single space."
- (string-trim (replace-regexp-in-string "[ \t\r\n]+" " " s)))
-
-(defun pulpie--page-blocks (html)
- "Split HTML into leaf blocks, returning (MARKUP NODES DOM).
-MARKUP is what the classifier sees: a div carrying `_item_id' and the block's
-raw text, mirroring pulpie's simplify output, since the model was distilled on
-that segmentation and not on the page's original tag names. NODES are the
-matching DOM elements, used to rebuild HTML around the kept blocks."
- (let* ((dom (with-temp-buffer
- (insert html)
- (libxml-parse-html-region (point-min) (point-max))))
- (root (or (car (dom-by-tag dom 'body)) dom))
- (markup '()) (nodes '()))
+(defun pulpie--blocks-from-dom (dom)
+ "Split DOM into leaf blocks, returning (MARKUP NODES).
+MARKUP is what the classifier sees: a div carrying `_item_id' and the
+block's raw text - the model was trained on this segmentation, not on the
+page's original tag names. NODES are the matching DOM elements."
+ (let ((root (or (car (dom-by-tag dom 'body)) dom))
+ (markup '()) (nodes '()))
(pulpie--walk root
(lambda (n _parent)
(when (pulpie--leaf-block-p n)
@@ -213,7 +165,7 @@ matching DOM elements, used to rebuild HTML around the kept blocks."
(length markup) (pulpie--escape text))
markup)
(push n nodes))))))
- (list (nreverse markup) (nreverse nodes) dom)))
+ (list (nreverse markup) (nreverse nodes))))
;;; Labels
@@ -252,14 +204,14 @@ matching DOM elements, used to rebuild HTML around the kept blocks."
(setq i (1+ i)))
(nreverse (cons (- (length ids) start) sizes))))
-(defun pulpie--classify (blocks &optional max-tokens)
+(defun pulpie--classify (blocks)
"Label every block 1 (main) or 0 (other), one request per chunk."
- (let* ((max-tokens (or max-tokens pulpie-max-tokens))
- (sizes (pulpie--block-sizes blocks))
+ (let* ((sizes (pulpie--block-sizes blocks))
(labels '()) (chunk '()) (chunk-tokens 0))
(cl-loop for block in blocks for size in sizes do
;; tokens so far + this block + one <|sep|> per block + BOS/EOS
- (when (and chunk (> (+ chunk-tokens size (length chunk) 3) max-tokens))
+ (when (and chunk (> (+ chunk-tokens size (length chunk) 3)
+ pulpie-max-tokens))
(setq labels (nconc labels (pulpie--label-chunk chunk)))
(setq chunk '() chunk-tokens 0))
(setq chunk (append chunk (list block))
@@ -269,44 +221,8 @@ matching DOM elements, used to rebuild HTML around the kept blocks."
;;; Rebuild
-(defun pulpie--prune (node)
- "Return a copy of NODE without `pulpie-noise-tags' descendants."
- (if (stringp node)
- node
- (cons (dom-tag node)
- (cons (dom-attributes node)
- (let (acc)
- (dolist (child (dom-children node))
- (cond ((stringp child) (push child acc))
- ((not (memq (dom-tag child) pulpie-noise-tags))
- (push (pulpie--prune child) acc))))
- (nreverse acc))))))
-
-(defun pulpie--self-closing (s)
- "Fix up libxml's XHTML-style self-closing tags in S.
-libxml prints every empty element as `<foo />', which HTML reads as an
-ordinary start tag - unbalanced inside a fragment. Void elements lose the
-slash; everything else gets an explicit close."
- (replace-regexp-in-string
- "<\\([a-zA-Z][a-zA-Z0-9-]*\\)\\( [^>]*\\)?/>"
- (lambda (match)
- (let ((tag (match-string 1 match))
- (attrs (string-trim-right (or (match-string 2 match) ""))))
- (if (member (downcase tag) pulpie-void-tags)
- (format "<%s%s>" tag attrs)
- (format "<%s%s></%s>" tag attrs tag))))
- s))
-
-(defun pulpie--serialize (node)
- "Serialize NODE to an HTML string, dropping noise tags."
- (with-temp-buffer
- (dom-print (pulpie--prune node))
- (let ((s (pulpie--self-closing (buffer-string))))
- ;; <pre> bodies stay verbatim; everything else is collapsed
- (if (string-match-p "<pre[ >]" s) s (pulpie--collapse s)))))
-
-(defun pulpie--merge-to-ancestors (nodes labels dom)
- "HTML for the kept blocks, merged up into their containing elements.
+(defun pulpie--kept-roots (nodes labels dom)
+ "Return the main-content roots for NODES labelled 1 by LABELS, in document order.
Classifying leaves loses the structure above them, so each kept block is
lifted to the largest ancestor that contains only kept blocks - that keeps
ul, table and section wrappers intact without dragging in neighbouring blocks
@@ -334,30 +250,43 @@ stable identity."
(lambda (n _parent)
(when (gethash n roots)
(remhash n roots)
- (let ((html (pulpie--serialize n)))
- (unless (string-empty-p html)
- (setq out (append out (list html))))))))
- out)))
-
-;;; Entry point
-
-(defun pulpie-main-content (html)
- "Return the main content of the document HTML as an HTML fragment.
-See `pulpie-main-html' for the networked version; this one is for a document
-you already have, e.g. an eww buffer's source."
- (pcase-let ((`(,markup ,nodes ,dom) (pulpie--page-blocks html)))
- (unless markup (error "No content blocks found"))
- (mapconcat #'identity
- (pulpie--merge-to-ancestors nodes (pulpie--classify markup) dom)
- "\n")))
+ (push n out))))
+ (nreverse out))))
+
+;;; Entry points
+
+(defun pulpie-readable-dom (dom)
+ "Drop-in replacement for `eww-readable-dom': return DOM as a `body'
+document containing only the main content, or nil if it has no
+classifiable blocks. Only `pulpie-server' is contacted."
+ (pcase-let ((`(,markup ,nodes) (pulpie--blocks-from-dom dom)))
+ (when markup
+ `(body nil ,@(pulpie--kept-roots nodes (pulpie--classify markup) dom)))))
+
+(defun pulpie--eww-readable-dom (orig dom)
+ "Around-advice for `eww-readable-dom': try pulpie on DOM, else ORIG.
+Fall back to the built-in heuristics if the pulpie server is unreachable
+or the page has no classifiable blocks."
+ (message "pulpie: DOM received from eww, classifying...")
+ (let ((start (float-time))
+ (result (condition-case err
+ (or (pulpie-readable-dom dom) (funcall orig dom))
+ (error
+ (message "pulpie: %s (using built-in readability)"
+ (error-message-string err))
+ (funcall orig dom)))))
+ (message "pulpie: returning DOM to eww (%.1fs)"
+ (- (float-time) start))
+ result))
;;;###autoload
-(defun pulpie-main-html (url)
- "Fetch URL and return its main content as an HTML fragment.
-The page's blocks are classified as main or other by the pulpie GGUF served at
-`pulpie-server'; kept blocks are merged up into their containing elements, so
-the fragment keeps headings, lists, tables and links as the page had them.
-Signal an error if the page cannot be fetched or has no classifiable blocks."
- (pulpie-main-content (pulpie--get url)))
+(define-minor-mode pulpie-eww-mode
+ "Read the main content of pages in eww with pulpie instead of heuristics.
+Wraps `eww-readable-dom' with `pulpie-readable-dom', so `eww-readable' and
+auto-readable pages (`eww-readable-urls') classify via `pulpie-server'."
+ :global t :group 'pulpie
+ (if pulpie-eww-mode
+ (advice-add 'eww-readable-dom :around #'pulpie--eww-readable-dom)
+ (advice-remove 'eww-readable-dom #'pulpie--eww-readable-dom)))
(provide 'pulpie)