pulpie-eww.el (12366B)
1 ;;; pulpie-eww.el --- eww-readable replacement engine, using pulpie BERT as classifier -*- lexical-binding: t; -*- 2 3 ;; Version: 1.0 4 ;; Package-Requires: ((emacs "30.1")) 5 6 ;;; Commentary: 7 8 ;; Enable with (pulpie-eww-mode 1): wraps `eww-readable-dom' with 9 ;; `pulpie-readable-dom', so eww's reader classifies main content with a 10 ;; pulpie classifier instead of heuristics. The DOM eww already parsed is 11 ;; used as-is; the only requests pulpie makes go to `pulpie-server', a 12 ;; llama-server (or compatible) exposing the model, e.g.: 13 ;; 14 ;; llama-server -m pulpie-orange-small-cls-f16.gguf --embeddings \ 15 ;; -b 8192 -ub 8192 -c 8192 16 ;; 17 ;; The DOM is split into leaf blocks, packed into the model's chunk template 18 ;; 19 ;; [BOS] block0 <|sep|> block1 <|sep|> ... <|sep|> [EOS] 20 ;; 21 ;; each <|sep|> row of /embedding carrying that block's [other, main] 22 ;; logits. Token ids come from /tokenize so they match the model's own 23 ;; tokenizer. 24 ;; 25 ;; Blocks are classified as leaves but emitted as whole subtrees: each kept 26 ;; block is lifted to the largest ancestor containing only kept blocks, so 27 ;; <ul>, <table> and section wrappers survive. 28 ;; 29 ;; In terms of performance overhead, it's around 0.1-0.8s when 30 ;; enabling reader mode on my machines. That's with the classifier 31 ;; running on CPU and not GPU. 32 33 ;;; Code: 34 35 (require 'cl-lib) 36 (require 'dom) 37 (require 'json) 38 (require 'pcase) 39 (require 'url) 40 41 (defgroup pulpie nil 42 "Main-content extraction with a pulpie classifier." 43 :group 'hypermedia) 44 45 (defcustom pulpie-server "http://localhost:9931" 46 "Base URL of the llama-server exposing the pulpie classifier." 47 :type 'string :group 'pulpie) 48 49 (defcustom pulpie-max-tokens 8192 50 "Token budget per request; must not exceed the server's context size." 51 :type 'integer :group 'pulpie) 52 53 (defconst pulpie-bos "<|begin_of_text|>" "Chunk prefix.") 54 (defconst pulpie-eos "<|end_of_text|>" "Chunk suffix.") 55 (defconst pulpie-sep "<|sep|>" "Block separator.") 56 (defconst pulpie-sep-token-id 128256 "Vocab id of <|sep|>.") 57 58 (defconst pulpie-block-tags 59 '(p li h1 h2 h3 h4 h5 h6 pre blockquote figcaption td th dd dt summary) 60 "Block-level tags whose text is classified (leaf elements only).") 61 62 ;;; HTTP 63 64 (defun pulpie--header-end () 65 "Return the position just past the HTTP header block." 66 (save-excursion 67 (goto-char (point-min)) 68 (if (re-search-forward "\r?\n\r?\n" nil t) (match-end 0) (point-min)))) 69 70 (defun pulpie--body (buffer url) 71 "Return BUFFER's body decoded as UTF-8, then kill BUFFER. 72 URL is only used in the error message. Signals on a 4xx/5xx status." 73 (with-current-buffer buffer 74 (unwind-protect 75 (let ((header-end (pulpie--header-end))) 76 (goto-char (point-min)) 77 (when (and (looking-at "HTTP/[0-9.]+ \([0-9]+\)") 78 (>= (string-to-number (match-string 1)) 400)) 79 (error "Request to %s failed (%s): %s" url (match-string 1) 80 (string-trim 81 (buffer-substring-no-properties header-end (point-max))))) 82 (let ((body (buffer-substring-no-properties header-end (point-max)))) 83 ;; url.el may leave the body as raw bytes 84 (if (multibyte-string-p body) 85 body 86 (decode-coding-string body 'utf-8)))) 87 (kill-buffer buffer)))) 88 89 (defun pulpie--post-json (url payload) 90 "POST PAYLOAD as JSON to URL and return the parsed response." 91 (let* ((url-show-status nil) 92 (url-request-method "POST") 93 (url-request-extra-headers 94 (list (cons "Content-Type" "application/json"))) 95 (url-request-data (encode-coding-string (json-encode payload) 'utf-8))) 96 (json-parse-string (pulpie--body (url-retrieve-synchronously url t nil 300) url) 97 ;; keys come back interned, hence `assq' below 98 :object-type 'alist :array-type 'list))) 99 100 (defun pulpie--tokenize (text) 101 "Return the token id list of TEXT, from the server's tokenizer." 102 (cdr (assq 'tokens (pulpie--post-json (concat pulpie-server "/tokenize") 103 (list (cons "content" text)))))) 104 105 (defun pulpie--logits (text) 106 "Return TEXT's per-token [other, main] logits as a list of rows." 107 (let ((first (car (pulpie--post-json (concat pulpie-server "/embedding") 108 (list (cons "content" text)))))) 109 (unless first (error "Empty /embedding response")) 110 (cdr (assq 'embedding first)))) 111 112 ;;; DOM -> blocks 113 114 (defun pulpie--element-p (node) 115 "Non-nil if NODE is a DOM element rather than a text node." 116 (and (consp node) (symbolp (car node)))) 117 118 (defun pulpie--walk (node fn) 119 "Call FN with every element under NODE in document order, plus its parent." 120 (letrec ((go (lambda (n parent) 121 (funcall fn n parent) 122 (dolist (child (dom-children n)) 123 (when (pulpie--element-p child) 124 (funcall go child n)))))) 125 (when (pulpie--element-p node) (funcall go node nil)))) 126 127 (defun pulpie--has-block-descendant-p (node) 128 "Non-nil if any descendant of NODE is one of `pulpie-block-tags'." 129 (catch 'found 130 (dolist (child (dom-children node)) 131 (when (pulpie--element-p child) 132 (when (memq (dom-tag child) pulpie-block-tags) (throw 'found t)) 133 (when (pulpie--has-block-descendant-p child) (throw 'found t)))) 134 nil)) 135 136 (defun pulpie--leaf-block-p (node) 137 "Non-nil if NODE is a block containing no other block." 138 (and (memq (dom-tag node) pulpie-block-tags) 139 (not (pulpie--has-block-descendant-p node)))) 140 141 (defun pulpie--escape (s) 142 "Escape S for the classifier markup. 143 The model was trained on markup where the apostrophe becomes ', so 144 this set of escapes, and no other, must be preserved: different escaping 145 yields different token ids." 146 (let ((s (replace-regexp-in-string "&" "&" s))) 147 (setq s (replace-regexp-in-string "<" "<" s)) 148 (setq s (replace-regexp-in-string ">" ">" s)) 149 (setq s (replace-regexp-in-string "\"" """ s)) 150 (replace-regexp-in-string "'" "'" s))) 151 152 (defun pulpie--blocks-from-dom (dom) 153 "Split DOM into leaf blocks, returning (MARKUP NODES). 154 MARKUP is what the classifier sees: a div carrying `_item_id' and the 155 block's raw text - the model was trained on this segmentation, not on the 156 page's original tag names. NODES are the matching DOM elements." 157 (let ((root (or (car (dom-by-tag dom 'body)) dom)) 158 (markup '()) (nodes '())) 159 (pulpie--walk root 160 (lambda (n _parent) 161 (when (pulpie--leaf-block-p n) 162 (let ((text (string-trim (or (dom-inner-text n) "")))) 163 (when (>= (length text) 2) 164 (push (format "<div _item_id=\"%d\">%s</div>" 165 (length markup) (pulpie--escape text)) 166 markup) 167 (push n nodes)))))) 168 (list (nreverse markup) (nreverse nodes)))) 169 170 ;;; Labels 171 172 (defun pulpie--argmax (row) 173 "Return the index of the largest value in ROW." 174 (let ((best 0) (best-value (car row)) (i 0)) 175 (dolist (v row) 176 (when (> v best-value) (setq best i best-value v)) 177 (setq i (1+ i))) 178 best)) 179 180 (defun pulpie--label-chunk (chunk) 181 "Classify CHUNK (a list of block markup strings), returning 0/1 per block." 182 (let* ((text (concat pulpie-bos 183 (mapconcat (lambda (b) (concat b pulpie-sep)) chunk "") 184 pulpie-eos)) 185 (ids (pulpie--tokenize text)) 186 (rows (pulpie--logits text)) 187 (sep-rows '()) (i 0)) 188 (dolist (tok ids) 189 (when (= tok pulpie-sep-token-id) (push i sep-rows)) 190 (setq i (1+ i))) 191 (setq sep-rows (nreverse sep-rows)) 192 (unless (= (length sep-rows) (length chunk)) 193 (error "%d <|sep|> rows for %d blocks" (length sep-rows) (length chunk))) 194 (mapcar (lambda (row-index) (pulpie--argmax (nth row-index rows))) sep-rows))) 195 196 (defun pulpie--block-sizes (blocks) 197 "Return each block's token count, from a single /tokenize round trip." 198 (let ((ids (pulpie--tokenize (mapconcat #'identity blocks pulpie-sep))) 199 (sizes '()) (start 0) (i 0)) 200 (dolist (tok ids) 201 (when (= tok pulpie-sep-token-id) 202 (push (- i start) sizes) 203 (setq start (1+ i))) 204 (setq i (1+ i))) 205 (nreverse (cons (- (length ids) start) sizes)))) 206 207 (defun pulpie--classify (blocks) 208 "Label every block 1 (main) or 0 (other), one request per chunk." 209 (let* ((sizes (pulpie--block-sizes blocks)) 210 (labels '()) (chunk '()) (chunk-tokens 0)) 211 (cl-loop for block in blocks for size in sizes do 212 ;; tokens so far + this block + one <|sep|> per block + BOS/EOS 213 (when (and chunk (> (+ chunk-tokens size (length chunk) 3) 214 pulpie-max-tokens)) 215 (setq labels (nconc labels (pulpie--label-chunk chunk))) 216 (setq chunk '() chunk-tokens 0)) 217 (setq chunk (append chunk (list block)) 218 chunk-tokens (+ chunk-tokens size))) 219 (when chunk (setq labels (nconc labels (pulpie--label-chunk chunk)))) 220 labels)) 221 222 ;;; Rebuild 223 224 (defun pulpie--kept-roots (nodes labels dom) 225 "Return the main-content roots for NODES labelled 1 by LABELS, in document order. 226 Classifying leaves loses the structure above them, so each kept block is 227 lifted to the largest ancestor that contains only kept blocks - that keeps 228 ul, table and section wrappers intact without dragging in neighbouring blocks 229 classified as other. DOM nodes are plain lists, so `eq' on the node is a 230 stable identity." 231 (let ((total (make-hash-table :test #'eq)) 232 (kept (make-hash-table :test #'eq)) 233 (parents (make-hash-table :test #'eq))) 234 (pulpie--walk dom (lambda (n parent) (when parent (puthash n parent parents)))) 235 (cl-loop for node in nodes for label in labels do 236 (let ((anc node)) 237 (while anc 238 (cl-incf (gethash anc total 0)) 239 (cl-incf (gethash anc kept 0) label) 240 (setq anc (gethash anc parents))))) 241 (let ((roots (make-hash-table :test #'eq)) 242 (out '())) 243 (cl-loop for node in nodes for label in labels when (= label 1) do 244 (let ((best node) (anc (gethash node parents))) 245 (while (and anc (not (memq (dom-tag anc) '(html body))) 246 (= (gethash anc total 0) (gethash anc kept 0))) 247 (setq best anc anc (gethash anc parents))) 248 (puthash best t roots))) 249 (pulpie--walk dom 250 (lambda (n _parent) 251 (when (gethash n roots) 252 (remhash n roots) 253 (push n out)))) 254 (nreverse out)))) 255 256 ;;; Entry points 257 258 (defun pulpie-readable-dom (dom) 259 "Drop-in replacement for `eww-readable-dom': return DOM as a `body' 260 document containing only the main content, or nil if it has no 261 classifiable blocks. Only `pulpie-server' is contacted." 262 (pcase-let ((`(,markup ,nodes) (pulpie--blocks-from-dom dom))) 263 (when markup 264 `(body nil ,@(pulpie--kept-roots nodes (pulpie--classify markup) dom))))) 265 266 (defun pulpie--eww-readable-dom (orig dom) 267 "Around-advice for `eww-readable-dom': try pulpie on DOM, else ORIG. 268 Fall back to the built-in heuristics if the pulpie server is unreachable 269 or the page has no classifiable blocks." 270 (message "pulpie: DOM received from eww, classifying...") 271 (let ((start (float-time)) 272 (result (condition-case err 273 (or (pulpie-readable-dom dom) (funcall orig dom)) 274 (error 275 (message "pulpie: %s (using built-in readability)" 276 (error-message-string err)) 277 (funcall orig dom))))) 278 (message "pulpie: returning DOM to eww (%.1fs)" 279 (- (float-time) start)) 280 result)) 281 282 ;;;###autoload 283 (define-minor-mode pulpie-eww-mode 284 "Read the main content of pages in eww with pulpie instead of heuristics. 285 Wraps `eww-readable-dom' with `pulpie-readable-dom', so `eww-readable' and 286 auto-readable pages (`eww-readable-urls') classify via `pulpie-server'." 287 :global t :group 'pulpie 288 (if pulpie-eww-mode 289 (advice-add 'eww-readable-dom :around #'pulpie--eww-readable-dom) 290 (advice-remove 'eww-readable-dom #'pulpie--eww-readable-dom))) 291 292 (provide 'pulpie-eww)