digest-html | Converts HTML to plaintext-with-links. |
| download: http://git.y1.nz/archives/digest-html.tar.gz | |
| README | Files | Log | Refs |
translate.c
1 /**
2 * This file defines how an HTML document is converted into plaintext-
3 * with-links.
4 */
5
6 #include <stdio.h> /* PRINTF, etc */
7 #include <stdlib.h> /* exit, EXIT_FAILURE, etc */
8 #include <ctype.h> /* isprint */
9
10 #include "translate.h"
11 #include "gumbo.h"
12 #include "okra.h"
13
14 /****** process nodes ******/
15
16 /* whitespace types */
17 #define SPACE_RESET -2 /* cancels previous spacing, allows spacing afterwards */
18 #define SPACE_CANCEL -1 /* cancels spacing before & prevents spacing after. */
19 #define SPACE_NONE 0 /* no space */
20 #define SPACE_SPACE 1 /* one ' ' */
21 #define SPACE_NEW_LINE 2 /* one '\n' */
22 #define SPACE_LINE_GAP 3 /* two '\n' */
23
24 /* raw bc they Actually Put, without flushing whitespace. */
25 /* should only be used in flush_space and their non-raw variants. */
26 #define PUTC_RAW(c) fputc(c, state->global->out)
27 #define PUTS_RAW(s) fputs(s, state->global->out)
28 #define PRINTF_RAW(...) fprintf(state->global->out, __VA_ARGS__)
29
30 /* macros for printing to output */
31 #define PUTC(c) { flush_space(state, c); PUTC_RAW(c); }
32 #define PUTS(s) { flush_space(state, s[0]); PUTS_RAW(s); }
33 #define PRINTF(...) { flush_space(state, '!'); PRINTF_RAW(__VA_ARGS__); }
34
35 #define PRINT_ATTRIBUTE(element, name) { \
36 const char *attr = get_attribute(element, name); \
37 if (attr != NULL) PUTS(get_attribute(element, name)); \
38 }
39
40 #define PUT_SPACE(kind) put_space(state, kind)
41
42 #define RECURSE { \
43 children = element->children; \
44 for (int i = 0; i < (int) children.length; i++) \
45 process_node(state, children.data[i]); \
46 }
47
48 void
49 put_space(EphemeralState *state, int spacing)
50 {
51 if (spacing == SPACE_RESET) {
52 state->spacing = SPACE_NONE;
53 return;
54 }
55
56 /* if it's spacing-cancel, ignore all new spacing */
57 if (state->spacing == SPACE_CANCEL)
58 return;
59
60 /* SPACE_NONE does nothing lol. */
61 if (spacing == SPACE_NONE)
62 return;
63
64 if (spacing == SPACE_LINE_GAP || spacing == SPACE_CANCEL) {
65 state->spacing = spacing;
66 return;
67 }
68
69 if (state->spacing == SPACE_NONE || state->spacing == SPACE_SPACE)
70 state->spacing = spacing;
71 }
72
73 void
74 put_indents(EphemeralState *state)
75 {
76 if (!state->verbatim)
77 for (int i = 0; i < state->indents; i++)
78 PUTC_RAW(' ');
79 }
80
81 void
82 flush_space(EphemeralState *state, char next)
83 {
84 /* THIS METHOD USES _RAW PRINTS, **NOT** the normal ones. */
85
86 // TODO need access to previous printed char....
87 switch (state->spacing) {
88 case SPACE_CANCEL: break; /* flushing a CANCEL just un-sets it. */
89 case SPACE_NONE: break;
90 case SPACE_SPACE:
91 // TODO other space-consumers?
92 if (next != ',' && next != ']' && next != '.')
93 PUTC_RAW(' ');
94 break;
95 case SPACE_NEW_LINE:
96 PUTS_RAW("\n");
97 put_indents(state);
98 break;
99 case SPACE_LINE_GAP:
100 PUTS_RAW("\n\n");
101 put_indents(state);
102 break;
103 }
104 state->spacing = SPACE_NONE;
105 }
106
107 // TODO is this even used? probably don't ever want to truncate anything.
108 void
109 print_truncated(EphemeralState *state, const char *string, int max_chars)
110 {
111 int j = 0;
112 char ch;
113 for (;;) {
114 ch = string[j];
115 if (ch == '\0') return;
116 if (j < max_chars) PUTC(ch);
117 j++;
118 }
119 }
120
121 const char *
122 get_attribute(GumboElement *element, const char *name)
123 {
124 GumboAttribute *attribute = gumbo_get_attribute(
125 &element->attributes,
126 name
127 );
128 if (attribute == NULL) return NULL;
129 return attribute->value;
130 }
131
132 void
133 print_link(EphemeralState *state, const char *link)
134 {
135 OkraUri *uri = NULL, *resolved = NULL;
136
137 if (link == NULL) {
138 link = "no link";
139 goto print_link_end;
140 }
141
142 if (state->global->base_uri == NULL) goto print_link_end;
143 uri = okra_parse(link);
144 if (uri == NULL) goto print_link_end;
145 resolved = okra_merge(state->global->base_uri, uri);
146 if (resolved != NULL) link = resolved->uri;
147
148 print_link_end:
149 PUTS(link);
150 if (uri != NULL) okra_free(uri);
151 if (resolved != NULL) okra_free(resolved);
152 }
153
154 void
155 process_element(EphemeralState *state, GumboElement *element) {
156 GumboVector children; /* used in RECURSE */
157
158 // TODO remove printf entirely...
159 if (state->global->debug >= 2)
160 PRINTF("[@");
161
162 /* These tags are defined in tag_enum.h in Gumbo. */
163 switch (element->tag) {
164 case GUMBO_TAG_HTML:
165 PUTS("HTML.");
166 PUT_SPACE(SPACE_NEW_LINE);
167 state->indents += 2;
168 RECURSE;
169 state->indents -= 2;
170 break;
171
172 case GUMBO_TAG_HEAD: break;
173 case GUMBO_TAG_TITLE:
174 PUTS("Title - ");
175 // TODO print the title (attribute?)
176 break;
177
178 /* tags that are omitted COMPLETELY: */
179 case GUMBO_TAG_SOURCE:
180 case GUMBO_TAG_STYLE:
181 case GUMBO_TAG_LINK:
182 case GUMBO_TAG_META:
183 case GUMBO_TAG_SCRIPT:
184 case GUMBO_TAG_FONT:
185 case GUMBO_TAG_SELECT:
186 case GUMBO_TAG_OPTION:
187 // PUTS("nr("); RECURSE; PUTS(")");
188 return;
189
190 case GUMBO_TAG_HEADER:
191 PUT_SPACE(SPACE_LINE_GAP);
192 state->indents -= 2;
193 RECURSE;
194 PUT_SPACE(SPACE_NEW_LINE);
195 PUTS("---");
196 PUT_SPACE(SPACE_LINE_GAP);
197 state->indents += 2;
198 break;
199 case GUMBO_TAG_FOOTER:
200 PUT_SPACE(SPACE_LINE_GAP);
201 state->indents += 2;
202 PUTS("---");
203 state->indents += 2;
204 PUT_SPACE(SPACE_NEW_LINE);
205 RECURSE;
206 state->indents -= 2;
207 state->indents -= 2;
208 PUT_SPACE(SPACE_LINE_GAP);
209 break;
210 case GUMBO_TAG_HR:
211 PUT_SPACE(SPACE_NEW_LINE);
212 PUTS("----------------------");
213 PUT_SPACE(SPACE_NEW_LINE);
214 RECURSE;
215 break;
216
217 /* Transparent tags: just show contents. */
218 case GUMBO_TAG_ARTICLE:
219 case GUMBO_TAG_SECTION:
220 case GUMBO_TAG_DIALOG:
221 case GUMBO_TAG_MAIN:
222 case GUMBO_TAG_LABEL:
223 case GUMBO_TAG_TIME:
224 case GUMBO_TAG_BODY:
225 case GUMBO_TAG_CENTER:
226 case GUMBO_TAG_NOSCRIPT: /* fuck javascript. */
227 RECURSE;
228 return;
229
230 case GUMBO_TAG_SUP:
231 PUTS("^");
232 RECURSE;
233 return;
234
235 case GUMBO_TAG_A:
236 PUTC('[');
237 PUT_SPACE(SPACE_CANCEL);
238 RECURSE;
239 PUT_SPACE(SPACE_CANCEL);
240 PUTS("](");
241 print_link(state, get_attribute(element, "href"));
242 PUTC(')');
243 // PUT_SPACE(SPACE_SPACE);
244 break;
245 case GUMBO_TAG_FORM:
246 const char *id = get_attribute(element, "id");
247
248 const char *target = get_attribute(element, "action");
249 const char *method = get_attribute(element, "method");
250 if (method == NULL) method = "GET";
251 if (target == NULL) target = "";
252
253
254 PUT_SPACE(SPACE_NEW_LINE);
255 PRINTF("[FORM method:%s id:%s ", method, id);
256 RECURSE;
257 PUTS("](");
258 print_link(state, target);
259 PUTC(')');
260
261 // TODO maybe, use children to create an editable url?
262 // TODO because lwl will have good support for "edit and
263 // TODO use a link", for example for removing tracking bs
264
265 // TODO OR, just "http-form://(URI)#(method, fields, etc)"?
266 break;
267 case GUMBO_TAG_INPUT:
268 PUTS("in:");
269 PRINT_ATTRIBUTE(element, "name");
270 // TODO handle undefined name more intentionally
271 PUT_SPACE(SPACE_SPACE);
272 RECURSE;
273 break;
274 case GUMBO_TAG_BR:
275 PUT_SPACE(SPACE_NEW_LINE);
276 RECURSE;
277 break;
278 case GUMBO_TAG_P:
279 // TODO indentation?
280 PUT_SPACE(SPACE_LINE_GAP);
281 state->indents += 2;
282 RECURSE;
283 state->indents -= 2;
284 PUT_SPACE(SPACE_LINE_GAP);
285 break;
286 case GUMBO_TAG_DIV:
287 PUT_SPACE(SPACE_NEW_LINE);
288 RECURSE;
289 PUT_SPACE(SPACE_NEW_LINE);
290 break;
291 case GUMBO_TAG_SPAN:
292 RECURSE;
293 PUT_SPACE(SPACE_SPACE);
294 break;
295 case GUMBO_TAG_DETAILS: RECURSE; break;
296 case GUMBO_TAG_SUMMARY: RECURSE; break;
297
298 case GUMBO_TAG_H1:
299 case GUMBO_TAG_H2:
300 case GUMBO_TAG_H3:
301 case GUMBO_TAG_H4:
302 case GUMBO_TAG_H5:
303 case GUMBO_TAG_H6:
304 PUTS("##");
305 PUT_SPACE(SPACE_SPACE);
306 RECURSE;
307 PUT_SPACE(SPACE_SPACE);
308 break;
309
310 case GUMBO_TAG_BUTTON:
311 /* we don't want to see buttons - just what's inside them. */
312 PUTS("[BUTTON");
313 RECURSE;
314 PUT_SPACE(SPACE_CANCEL);
315 PUTS("]");
316 break;
317
318 case GUMBO_TAG_CODE: /* FALLTHROUGH */
319 case GUMBO_TAG_PRE:
320 state->verbatim = 1;
321 PUT_SPACE(SPACE_SPACE);
322 RECURSE;
323 PUT_SPACE(SPACE_SPACE);
324 state->verbatim = 0;
325 break;
326
327 case GUMBO_TAG_EM: /* FALLTHROUGH */
328 case GUMBO_TAG_I:
329 PUTS("/");
330 RECURSE;
331 PUTS("/");
332 break;
333
334 case GUMBO_TAG_U:
335 PUTS("__");
336 RECURSE;
337 PUTS("__");
338 break;
339
340 case GUMBO_TAG_STRONG: /* FALLTHROUGH */
341 case GUMBO_TAG_B:
342 PUTS("**");
343 RECURSE;
344 PUTS("**");
345 break;
346
347 case GUMBO_TAG_IMG:
348 PUTS("[img:");
349 PRINT_ATTRIBUTE(element, "alt"); // TODO or title...
350 PUTS("](");
351 print_link(state, get_attribute(element, "src"));
352 PUTC(')');
353 // RECURSE;
354 break;
355 case GUMBO_TAG_VIDEO:
356 PUTS("[video](");
357 print_link(state, get_attribute(element, "src"));
358 PUTS(")");
359 break;
360 case GUMBO_TAG_AUDIO:
361 PUTS("[audio](");
362 print_link(state, get_attribute(element, "src"));
363 PUTS(")");
364 break;
365 case GUMBO_TAG_SVG:
366 // PUTS("(svg)");
367 /* do not recurse, svg garbage is inside */
368 // TODO: process into a downloadable SVG (as link)?
369 break;
370 case GUMBO_TAG_PICTURE:
371 // PUTS("(pic)");
372 // PUTS("[Picture: ");
373 // PRINT_ATTRIBUTE(element, "title"); // TODO: or title...
374 // PUTS("](<TODO: extract internal source element?>)");
375 // RECURSE;
376 break;
377
378 case GUMBO_TAG_TEXTAREA:
379 // PUTS("(TEXTAREA)[_](<text area input...>)");
380 // RECURSE;
381 break;
382 case GUMBO_TAG_NAV:
383 RECURSE;
384 break;
385
386 case GUMBO_TAG_OL:
387 case GUMBO_TAG_UL:
388 int old_ordered = state->list_ordered;
389 int old_index = state->list_index;
390
391 state->list_ordered = (element->tag == GUMBO_TAG_OL);
392 state->list_index = 1;
393
394 PUT_SPACE(SPACE_NEW_LINE);
395
396 state->indents += 2;
397 RECURSE;
398 state->indents -= 2;
399
400 PUT_SPACE(SPACE_NEW_LINE);
401
402 state->list_index = old_index;
403 state->list_ordered = old_ordered;
404 break;
405
406 case GUMBO_TAG_LI:
407 PUT_SPACE(SPACE_SPACE);
408 if (state->list_ordered) {
409 PUTS("[");
410 PRINTF("%d", state->list_index);
411 PUTS(")");
412 } else {
413 PUTS("-");
414 }
415 PUT_SPACE(SPACE_SPACE);
416
417 state->indents += 2;
418 RECURSE;
419 state->indents -= 2;
420
421 state->list_index++;
422
423 PUT_SPACE(state->list_ordered ? SPACE_LINE_GAP : SPACE_NEW_LINE);
424
425 break;
426 case GUMBO_TAG_IFRAME:
427 PRINTF("(IFRAME)");
428 RECURSE;
429 break;
430 case GUMBO_TAG_TABLE:
431 PUT_SPACE(SPACE_NEW_LINE);
432 RECURSE;
433 PUT_SPACE(SPACE_LINE_GAP);
434 break;
435 case GUMBO_TAG_TBODY: RECURSE; break;
436 case GUMBO_TAG_TR:
437 PUT_SPACE(SPACE_NEW_LINE);
438 RECURSE;
439 PUT_SPACE(SPACE_LINE_GAP);
440 break;
441 case GUMBO_TAG_THEAD: /* FALLTHROUGH */
442 case GUMBO_TAG_TH:
443 case GUMBO_TAG_TD:
444 PUT_SPACE(SPACE_SPACE);
445 RECURSE;
446 PUT_SPACE(SPACE_SPACE);
447 break;
448 case GUMBO_TAG_DL:
449 PUT_SPACE(SPACE_NEW_LINE);
450 RECURSE;
451 PUT_SPACE(SPACE_NEW_LINE);
452 break;
453 case GUMBO_TAG_DD:
454 RECURSE;
455 break;
456 case GUMBO_TAG_DT:
457 PUT_SPACE(SPACE_NEW_LINE);
458 RECURSE;
459 PUTS(":");
460 PUT_SPACE(SPACE_SPACE);
461 break;
462 default:
463 const char *tagname = gumbo_normalized_tagname(element->tag);
464 if (state->global->debug >= 1 && tagname[0] != '\0') PUTS("[");
465 RECURSE;
466 if (state->global->debug && tagname[0] != '\0')
467 PRINTF("](unknown tag- '%s')", tagname);
468 break;
469 }
470
471 if (state->global->debug >= 2)
472 PRINTF("](processing %s)", gumbo_normalized_tagname(element->tag));
473 }
474
475 void
476 print_text(EphemeralState *state, GumboText gt)
477 {
478 if (state->verbatim) {
479 // TODO handle the surrounding space...
480 PRINTF("%s", gt.text);
481 return;
482 }
483
484 /* replace all whitespace with a single space char */
485 const char *text = gt.text;
486 char ch = 'a';
487 int i = 0;
488 for (;;) {
489 ch = text[i++];
490
491 switch (ch) {
492 case '\0': return;
493 case '\n':
494 case ' ':
495 case '\t': PUT_SPACE(SPACE_SPACE); break;
496 case '[': PUTS("\\["); break;
497 case ']': PUTS("\\]"); break;
498 case '(': PUTS("\\("); break;
499 case ')': PUTS("\\)"); break;
500 default:
501 PUTC(ch);
502 break;
503 }
504 }
505 }
506
507 void
508 process_node(EphemeralState *state, GumboNode *node)
509 {
510 switch (node->type) {
511 case GUMBO_NODE_DOCUMENT:
512 PUTS("Bad node: do not pass document here." "\n");
513 exit(EXIT_FAILURE);
514
515 case GUMBO_NODE_ELEMENT:
516 process_element(state, &node->v.element);
517 break;
518
519 case GUMBO_NODE_TEXT:
520 print_text(state, node->v.text);
521 break;
522
523 case GUMBO_NODE_CDATA:
524 PUTS("C>");
525 print_text(state, node->v.text);
526 PUTS("<C");
527 break;
528
529 case GUMBO_NODE_WHITESPACE:
530 print_text(state, node->v.text); break;
531 break;
532
533 default: /* COMMENT, TEMPLATE, etc */
534 /* ignore completely. */
535 break;
536 }
537
538 }
This webpage is intended to be an accessible preview of this repository. To get a fuller picture, clone it and use the git CLI.