pulpie-eww.el

replacing eww-readable with pulpie-orange-small
Log | Files | Refs | LICENSE

commit 133712b6981c79c89f0251e4f131e9049a52f1db
parent 2d76af7f779fdf79a9a87bbc58b86789d79062a7
Author: Vineet Kumar <git@vineetk.net>
Date:   Fri, 20 Nov 2026 02:15:27 +0100

rewrite by reusing eww functions

so now only network requests it makes is to `pulpie-server'.
and its dom stuff is much simpler too.

Diffstat:
Mpulpie.el | 231++++++++++++++++++++++++++++---------------------------------------------------
1 file changed, 80 insertions(+), 151 deletions(-)

diff --git a/pulpie.el b/pulpie.el @@ -5,32 +5,30 @@ ;;; Commentary: -;; One entry point: -;; -;; (pulpie-main-html "https://example.com/article") -;; => "<main><h1>...</h1><p>...</p></main>" -;; -;; It returns the page's main content as an HTML fragment - navigation, -;; sidebars, comments and footers dropped - intended as the reader step in -;; front of eww, which otherwise shows the whole document. -;; -;; Classification runs on a llama-server serving a pulpie GGUF: +;; Enable with (pulpie-eww-mode 1): wraps `eww-readable-dom' with +;; `pulpie-readable-dom', so eww's reader classifies main content with a +;; pulpie classifier instead of heuristics. The DOM eww already parsed is +;; used as-is; the only requests pulpie makes go to `pulpie-server', a +;; llama-server (or compatible) exposing the model, e.g.: ;; ;; llama-server -m pulpie-orange-small-cls-f16.gguf --embeddings \ -;; -b 8192 -ub 8192 -c 8192 --port 8888 +;; -b 8192 -ub 8192 -c 8192 ;; -;; How it works: the page is split into leaf blocks, packed into the model's -;; chunk template +;; The DOM is split into leaf blocks, packed into the model's chunk template ;; ;; [BOS] block0 <|sep|> block1 <|sep|> ... <|sep|> [EOS] ;; -;; and every <|sep|> row of /embedding carries that block's [other, main] -;; logits. Token ids come from the server's /tokenize endpoint, so they match -;; the tokenizer baked into the GGUF exactly. +;; each <|sep|> row of /embedding carrying that block's [other, main] +;; logits. Token ids come from /tokenize so they match the model's own +;; tokenizer. ;; ;; Blocks are classified as leaves but emitted as whole subtrees: each kept ;; block is lifted to the largest ancestor containing only kept blocks, so ;; <ul>, <table> and section wrappers survive. +;; +;; In terms of performance overhead, it's around 0.1-0.8s when +;; enabling reader mode on my machines. That's with the classifier +;; running on CPU and not GPU. ;;; Code: @@ -39,21 +37,17 @@ (require 'json) (require 'pcase) (require 'url) -(require 'url-http) (defgroup pulpie nil - "Main-content extraction with a pulpie GGUF behind llama-server." + "Main-content extraction with a pulpie classifier." :group 'hypermedia) -;; (defcustom pulpie-server "http://localhost:8888" -;; "Base URL of the llama-server serving a pulpie GGUF." -;; :type 'string :group 'pulpie) (defcustom pulpie-server "http://llm.demiurge.epistemia/upstream/pulpie-orange-small" - "Base URL of the llama-server serving a pulpie GGUF." + "Base URL of the llama-server exposing the pulpie classifier." :type 'string :group 'pulpie) (defcustom pulpie-max-tokens 8192 - "Token budget per request; must not exceed the server's -b/-ub/-c value." + "Token budget per request; must not exceed the server's context size." :type 'integer :group 'pulpie) (defconst pulpie-bos "<|begin_of_text|>" "Chunk prefix.") @@ -65,15 +59,6 @@ '(p li h1 h2 h3 h4 h5 h6 pre blockquote figcaption td th dd dt summary) "Block-level tags whose text is classified (leaf elements only).") -(defconst pulpie-noise-tags - '(script style noscript iframe template svg canvas form) - "Tags dropped when merged blocks are serialized.") - -(defconst pulpie-void-tags - '("area" "base" "br" "col" "embed" "hr" "img" "input" "link" "meta" - "source" "track" "wbr") - "HTML elements that must not be printed with a closing tag.") - ;;; HTTP (defun pulpie--header-end () @@ -83,55 +68,30 @@ (if (re-search-forward "\r?\n\r?\n" nil t) (match-end 0) (point-min)))) (defun pulpie--body (buffer url) - "Return BUFFER's decoded body as a string, then kill BUFFER. + "Return BUFFER's body decoded as UTF-8, then kill BUFFER. URL is only used in the error message. Signals on a 4xx/5xx status." (with-current-buffer buffer (unwind-protect - (let* ((status (save-excursion - (goto-char (point-min)) - (when (looking-at "HTTP/[0-9.]+ \\([0-9]+\\)") - (string-to-number (match-string 1))))) - (header-end (pulpie--header-end)) - (charset (save-excursion - (goto-char (point-min)) - (when (re-search-forward - "^Content-Type:.*?charset=\\([^; \r\n]+\\)" - header-end t) - (match-string 1)))) - ;; no charset is UTF-8, not the ISO-8859-1 of an old RFC - (coding (let ((sym (and charset (ignore-errors (intern charset))))) - (if (and sym (coding-system-p sym)) sym 'utf-8)))) - (when (and status (>= status 400)) - (error "Request to %s failed (%s): %s" - url status + (let ((header-end (pulpie--header-end))) + (goto-char (point-min)) + (when (and (looking-at "HTTP/[0-9.]+ \([0-9]+\)") + (>= (string-to-number (match-string 1)) 400)) + (error "Request to %s failed (%s): %s" url (match-string 1) (string-trim (buffer-substring-no-properties header-end (point-max))))) - (let ((body (buffer-substring-no-properties (pulpie--header-end) (point-max)))) - ;; url.el leaves the body as raw bytes (a unibyte string); decode - ;; them here. decode-coding-region on the buffer is a no-op. + (let ((body (buffer-substring-no-properties header-end (point-max)))) + ;; url.el may leave the body as raw bytes (if (multibyte-string-p body) body - (decode-coding-string body coding)))) + (decode-coding-string body 'utf-8)))) (kill-buffer buffer)))) -(defun pulpie--get (url) - "Fetch URL and return its decoded body." - (let ((url-user-agent url-user-agent) - (url-mime-accept-string "text/html,application/xhtml+xml,*/*") - (url-show-status nil) - (url-request-method "GET") - (url-request-extra-headers - (list (cons "User-Agent" pulpie-user-agent) - (cons "Accept" "text/html,application/xhtml+xml,*/*")))) - (pulpie--body (url-retrieve-synchronously url t nil 60) url))) - (defun pulpie--post-json (url payload) "POST PAYLOAD as JSON to URL and return the parsed response." (let* ((url-show-status nil) (url-request-method "POST") (url-request-extra-headers - (list (cons "Content-Type" "application/json") - (cons "User-Agent" pulpie-user-agent))) + (list (cons "Content-Type" "application/json"))) (url-request-data (encode-coding-string (json-encode payload) 'utf-8))) (json-parse-string (pulpie--body (url-retrieve-synchronously url t nil 300) url) ;; keys come back interned, hence `assq' below @@ -180,30 +140,22 @@ URL is only used in the error message. Signals on a 4xx/5xx status." (defun pulpie--escape (s) "Escape S for the classifier markup. -Mirrors Python's html.escape(s, quote=True), which is what the model was -verified against - including the apostrophe, because an apostrophe becomes -&#x27; and that changes the token ids the classifier sees." +The model was trained on markup where the apostrophe becomes &#x27;, so +this set of escapes, and no other, must be preserved: different escaping +yields different token ids." (let ((s (replace-regexp-in-string "&" "&amp;" s))) (setq s (replace-regexp-in-string "<" "&lt;" s)) (setq s (replace-regexp-in-string ">" "&gt;" s)) (setq s (replace-regexp-in-string "\"" "&quot;" s)) (replace-regexp-in-string "'" "&#x27;" s))) -(defun pulpie--collapse (s) - "Collapse every whitespace run in S to a single space." - (string-trim (replace-regexp-in-string "[ \t\r\n]+" " " s))) - -(defun pulpie--page-blocks (html) - "Split HTML into leaf blocks, returning (MARKUP NODES DOM). -MARKUP is what the classifier sees: a div carrying `_item_id' and the block's -raw text, mirroring pulpie's simplify output, since the model was distilled on -that segmentation and not on the page's original tag names. NODES are the -matching DOM elements, used to rebuild HTML around the kept blocks." - (let* ((dom (with-temp-buffer - (insert html) - (libxml-parse-html-region (point-min) (point-max)))) - (root (or (car (dom-by-tag dom 'body)) dom)) - (markup '()) (nodes '())) +(defun pulpie--blocks-from-dom (dom) + "Split DOM into leaf blocks, returning (MARKUP NODES). +MARKUP is what the classifier sees: a div carrying `_item_id' and the +block's raw text - the model was trained on this segmentation, not on the +page's original tag names. NODES are the matching DOM elements." + (let ((root (or (car (dom-by-tag dom 'body)) dom)) + (markup '()) (nodes '())) (pulpie--walk root (lambda (n _parent) (when (pulpie--leaf-block-p n) @@ -213,7 +165,7 @@ matching DOM elements, used to rebuild HTML around the kept blocks." (length markup) (pulpie--escape text)) markup) (push n nodes)))))) - (list (nreverse markup) (nreverse nodes) dom))) + (list (nreverse markup) (nreverse nodes)))) ;;; Labels @@ -252,14 +204,14 @@ matching DOM elements, used to rebuild HTML around the kept blocks." (setq i (1+ i))) (nreverse (cons (- (length ids) start) sizes)))) -(defun pulpie--classify (blocks &optional max-tokens) +(defun pulpie--classify (blocks) "Label every block 1 (main) or 0 (other), one request per chunk." - (let* ((max-tokens (or max-tokens pulpie-max-tokens)) - (sizes (pulpie--block-sizes blocks)) + (let* ((sizes (pulpie--block-sizes blocks)) (labels '()) (chunk '()) (chunk-tokens 0)) (cl-loop for block in blocks for size in sizes do ;; tokens so far + this block + one <|sep|> per block + BOS/EOS - (when (and chunk (> (+ chunk-tokens size (length chunk) 3) max-tokens)) + (when (and chunk (> (+ chunk-tokens size (length chunk) 3) + pulpie-max-tokens)) (setq labels (nconc labels (pulpie--label-chunk chunk))) (setq chunk '() chunk-tokens 0)) (setq chunk (append chunk (list block)) @@ -269,44 +221,8 @@ matching DOM elements, used to rebuild HTML around the kept blocks." ;;; Rebuild -(defun pulpie--prune (node) - "Return a copy of NODE without `pulpie-noise-tags' descendants." - (if (stringp node) - node - (cons (dom-tag node) - (cons (dom-attributes node) - (let (acc) - (dolist (child (dom-children node)) - (cond ((stringp child) (push child acc)) - ((not (memq (dom-tag child) pulpie-noise-tags)) - (push (pulpie--prune child) acc)))) - (nreverse acc)))))) - -(defun pulpie--self-closing (s) - "Fix up libxml's XHTML-style self-closing tags in S. -libxml prints every empty element as `<foo />', which HTML reads as an -ordinary start tag - unbalanced inside a fragment. Void elements lose the -slash; everything else gets an explicit close." - (replace-regexp-in-string - "<\\([a-zA-Z][a-zA-Z0-9-]*\\)\\( [^>]*\\)?/>" - (lambda (match) - (let ((tag (match-string 1 match)) - (attrs (string-trim-right (or (match-string 2 match) "")))) - (if (member (downcase tag) pulpie-void-tags) - (format "<%s%s>" tag attrs) - (format "<%s%s></%s>" tag attrs tag)))) - s)) - -(defun pulpie--serialize (node) - "Serialize NODE to an HTML string, dropping noise tags." - (with-temp-buffer - (dom-print (pulpie--prune node)) - (let ((s (pulpie--self-closing (buffer-string)))) - ;; <pre> bodies stay verbatim; everything else is collapsed - (if (string-match-p "<pre[ >]" s) s (pulpie--collapse s))))) - -(defun pulpie--merge-to-ancestors (nodes labels dom) - "HTML for the kept blocks, merged up into their containing elements. +(defun pulpie--kept-roots (nodes labels dom) + "Return the main-content roots for NODES labelled 1 by LABELS, in document order. Classifying leaves loses the structure above them, so each kept block is lifted to the largest ancestor that contains only kept blocks - that keeps ul, table and section wrappers intact without dragging in neighbouring blocks @@ -334,30 +250,43 @@ stable identity." (lambda (n _parent) (when (gethash n roots) (remhash n roots) - (let ((html (pulpie--serialize n))) - (unless (string-empty-p html) - (setq out (append out (list html)))))))) - out))) - -;;; Entry point - -(defun pulpie-main-content (html) - "Return the main content of the document HTML as an HTML fragment. -See `pulpie-main-html' for the networked version; this one is for a document -you already have, e.g. an eww buffer's source." - (pcase-let ((`(,markup ,nodes ,dom) (pulpie--page-blocks html))) - (unless markup (error "No content blocks found")) - (mapconcat #'identity - (pulpie--merge-to-ancestors nodes (pulpie--classify markup) dom) - "\n"))) + (push n out)))) + (nreverse out)))) + +;;; Entry points + +(defun pulpie-readable-dom (dom) + "Drop-in replacement for `eww-readable-dom': return DOM as a `body' +document containing only the main content, or nil if it has no +classifiable blocks. Only `pulpie-server' is contacted." + (pcase-let ((`(,markup ,nodes) (pulpie--blocks-from-dom dom))) + (when markup + `(body nil ,@(pulpie--kept-roots nodes (pulpie--classify markup) dom))))) + +(defun pulpie--eww-readable-dom (orig dom) + "Around-advice for `eww-readable-dom': try pulpie on DOM, else ORIG. +Fall back to the built-in heuristics if the pulpie server is unreachable +or the page has no classifiable blocks." + (message "pulpie: DOM received from eww, classifying...") + (let ((start (float-time)) + (result (condition-case err + (or (pulpie-readable-dom dom) (funcall orig dom)) + (error + (message "pulpie: %s (using built-in readability)" + (error-message-string err)) + (funcall orig dom))))) + (message "pulpie: returning DOM to eww (%.1fs)" + (- (float-time) start)) + result)) ;;;###autoload -(defun pulpie-main-html (url) - "Fetch URL and return its main content as an HTML fragment. -The page's blocks are classified as main or other by the pulpie GGUF served at -`pulpie-server'; kept blocks are merged up into their containing elements, so -the fragment keeps headings, lists, tables and links as the page had them. -Signal an error if the page cannot be fetched or has no classifiable blocks." - (pulpie-main-content (pulpie--get url))) +(define-minor-mode pulpie-eww-mode + "Read the main content of pages in eww with pulpie instead of heuristics. +Wraps `eww-readable-dom' with `pulpie-readable-dom', so `eww-readable' and +auto-readable pages (`eww-readable-urls') classify via `pulpie-server'." + :global t :group 'pulpie + (if pulpie-eww-mode + (advice-add 'eww-readable-dom :around #'pulpie--eww-readable-dom) + (advice-remove 'eww-readable-dom #'pulpie--eww-readable-dom))) (provide 'pulpie)