Dave Jarvis' Repositories
git clone https://repo.autonoma.ca/repo/keenwrite.git
Add DocumentParser to replace JSoup functionality, add KeenQuotes library
| Author | DaveJarvis <email> |
|---|---|
| Date | 2021-06-27 14:18:15 GMT-0700 |
| Commit | 5cd6619ab0869afa39743db759b00e88982eb0b6 |
| Parent | be43467 |
| Delta | 94 lines added, 299 lines removed, 205-line decrease |
src/test/resources/com/keenwrite/quotes/smartypants.txt
| -# ######################################################################## | ||
| -# Decades | ||
| -# ######################################################################## | ||
| -The Roaring '20s had the best music, no? | ||
| -The Roaring '20s had the best music, no? | ||
| - | ||
| -Took place in '04, yes'm! | ||
| -Took place in '04, yes'm! | ||
| - | ||
| -# ######################################################################## | ||
| -# Inside contractions (no leading/trailing apostrophes) | ||
| -# ######################################################################## | ||
| -I don't like it: I love's it! | ||
| -I don't like it: I love's it! | ||
| - | ||
| -We'd've thought that pancakes'll be sweeter there. | ||
| -We'd've thought that pancakes'll be sweeter there. | ||
| - | ||
| -She'd be coming o'er when the horse'd gone to pasture... | ||
| -She'd be coming o'er when the horse'd gone to pasture... | ||
| - | ||
| -# ######################################################################## | ||
| -# Beginning contractions (leading apostrophes) | ||
| -# ######################################################################## | ||
| -'Twas and 'tis whate'er lay 'twixt dawn and dusk 'n River Styx. | ||
| -'Twas and 'tis whate'er lay 'twixt dawn and dusk 'n River Styx. | ||
| - | ||
| -# ######################################################################## | ||
| -# Ending contractions (trailing apostrophes) | ||
| -# ######################################################################## | ||
| -Didn' get th' message. | ||
| -Didn' get th' message. | ||
| - | ||
| -Namsayin', y'know what I'ma sayin'? | ||
| -Namsayin', y'know what I'ma sayin'? | ||
| - | ||
| -# ######################################################################## | ||
| -# Outside contractions (leading and trailing, no middle) | ||
| -# ######################################################################## | ||
| -Salt 'n' vinegar, fish-'n'-chips, sugar 'n' spice! | ||
| -Salt 'n' vinegar, fish-'n'-chips, sugar 'n' spice! | ||
| - | ||
| -# ######################################################################## | ||
| -# Primes (single, double) | ||
| -# ######################################################################## | ||
| -She stood 5\'7\". | ||
| -She stood 5′7″. | ||
| - | ||
| -# No space after the feet sign. | ||
| -It's 4'11" away. | ||
| -It's 4′11″ away. | ||
| - | ||
| -Alice's friend is 6'3" tall. | ||
| -Alice's friend is 6′3″ tall. | ||
| - | ||
| -Bob's table is 5'' × 4''. | ||
| -Bob's table is 5″ × 4″. | ||
| - | ||
| -What's this -5.5'' all about? | ||
| -What's this -5.5″ all about? | ||
| - | ||
| -+7.9'' is weird. | ||
| -+7.9″ is weird. | ||
| - | ||
| -Foolscap? Naw, I use 11.5"x14.25" paper! | ||
| -Foolscap? Naw, I use 11.5″x14.25″ paper! | ||
| - | ||
| -An angular measurement, 3° 5' 30" means 3 degs, 5 arcmins, and 30 arcsecs. | ||
| -An angular measurement, 3° 5′ 30″ means 3 degs, 5 arcmins, and 30 arcsecs. | ||
| - | ||
| -# ######################################################################## | ||
| -# Backticks (left and right double quotes) | ||
| -# ######################################################################## | ||
| -``I am Sam'' | ||
| -“I am Sam” | ||
| - | ||
| -``Sam's away today'' | ||
| -“Sam's away today” | ||
| - | ||
| -``Sam's gone! | ||
| -“Sam's gone! | ||
| - | ||
| -``5'10" tall 'e was!'' | ||
| -“5′10″ tall 'e was!” | ||
| - | ||
| -# ######################################################################## | ||
| -# Consecutive quotes | ||
| -# ######################################################################## | ||
| -"'I'm trouble.'" | ||
| -“‘I'm trouble.’” | ||
| - | ||
| -'"Trouble's my name."' | ||
| -‘“Trouble's my name.“‘ | ||
| - | ||
| -# ######################################################################## | ||
| -# Escaped quotes | ||
| -# ######################################################################## | ||
| -\"What?\" | ||
| -“What?” | ||
| - | ||
| -# ######################################################################## | ||
| -# Double quotes | ||
| -# ######################################################################## | ||
| -"I am Sam" | ||
| -“I am Sam” | ||
| - | ||
| -"...even better!" | ||
| -“...even better!” | ||
| - | ||
| -"It was so," said he. | ||
| -“It was so,” said he. | ||
| - | ||
| -"She said, 'Llamas'll languish, they'll-- | ||
| -“She said, ‘Llamas'll languish, they'll-- | ||
| - | ||
| -With "air quotes" in the middle. | ||
| -With “air quotes” in the middle. | ||
| - | ||
| -With--"air quotes"--and dashes. | ||
| -With--“air quotes”--and dashes. | ||
| - | ||
| -"Not "quite" what you expected?" | ||
| -“Not “quite” what you expected?” | ||
| - | ||
| -# ######################################################################## | ||
| -# Nested quotations | ||
| -# ######################################################################## | ||
| -"'Here I am,' said Sam" | ||
| -“‘Here I am,’ said Sam” | ||
| - | ||
| -'"Here I am," said Sam' | ||
| -‘“Here I am,”, said Sam’ | ||
| - | ||
| -'Hello, "Dr. Brown," what's your real name?' | ||
| -‘Hello, “Dr. Brown,” what's your real name?’ | ||
| - | ||
| -"'Twas, t'wasn't thy name, 'twas it?" said Jim "the Barber" Brown. | ||
| -“'Twas, t'wasn't thy name, 'twas it?” said Jim “the Barber” Brown. | ||
| - | ||
| -# ######################################################################## | ||
| -# Single quotes | ||
| -# ######################################################################## | ||
| -'I am Sam' | ||
| -‘I am Sam’ | ||
| - | ||
| -'It was so,' said he. | ||
| -‘It was so,’ said he. | ||
| - | ||
| -'...even better!' | ||
| -‘...even better!’ | ||
| - | ||
| -With 'quotes' in the middle. | ||
| -With ‘quotes’ in the middle. | ||
| - | ||
| -With--'imaginary'--dashes. | ||
| -With--‘imaginary’--dashes. | ||
| - | ||
| -'Not 'quite' what you expected?' | ||
| -‘Not ‘quite’ what you expected?’ | ||
| - | ||
| -''Cause I don't like it, 's why,' said Pat. | ||
| -‘'Cause I don't like it, 's why,’ said Pat. | ||
| - | ||
| -'It's a beautiful day!' | ||
| -‘It's a beautiful day!’ | ||
| - | ||
| -'He said, 'Thinkin'.' | ||
| -‘He said, ‘Thinkin’.’ | ||
| - | ||
| -# ######################################################################## | ||
| -# Possessives | ||
| -# ######################################################################## | ||
| -Sam's Sams' and the Ross's roses' thorns were prickly. | ||
| -Sam's Sams' and the Ross's roses' thorns were prickly. | ||
| - | ||
| -# ######################################################################## | ||
| -# Mixed | ||
| -# ######################################################################## | ||
| -"I heard she said, 'That's Sam's'," said the Sams' cat. | ||
| -“I heard she said, ‘That's Sam's’,” said the Sams' cat. | ||
| - | ||
| -"'Janes' said, ''E'll be spooky, Sam's son with the jack-o'-lantern!'" said the O'Mally twins'---y'know---ghosts in unison. | ||
| -“‘Janes' said, ‘'E'll be spooky, Sam's son with the jack-o'-lantern!’” said the O'Mally twins'---y'know---ghosts in unison. | ||
| - | ||
| -'He's at Sams' | ||
| -‘He' at Sams’ | ||
| - | ||
| -\"Hello!\" | ||
| -“Hello!” | ||
| - | ||
| -ma'am | ||
| -ma'am | ||
| - | ||
| -'Twas midnight | ||
| -'Twas midnight | ||
| - | ||
| -\"Hello,\" said the spider. \"'Shelob' is my name.\" | ||
| -“Hello,” said the spider. “‘Shelob’ is my name.” | ||
| - | ||
| -'A', 'B', and 'C' are letters. | ||
| -‘A’ ‘B’ and ‘C’ are letters. | ||
| - | ||
| -'Oak,' 'elm,' and 'beech' are names of trees. So is 'pine.' | ||
| -‘Oak,’ ‘elm,’ and ‘beech’ are names of trees. So is ‘pine.’ | ||
| - | ||
| -'He said, \"I want to go.\"' Were you alive in the 70's? | ||
| -‘He said, “I want to go.”’ Were you alive in the 70's? | ||
| - | ||
| -\"That's a 'magic' sock.\" | ||
| -“That's a ‘magic’ sock.” | ||
| - | ||
| -Website! Company Name, Inc. (\"Company Name\" or \"Company\") recommends reading the following terms and conditions, carefully: | ||
| -Website! Company Name, Inc. (“Company Name” or “Company”) recommends reading the following terms and conditions, carefully: | ||
| - | ||
| -Website! Company Name, Inc. ('Company Name' or 'Company') recommends reading the following terms and conditions, carefully: | ||
| -Website! Company Name, Inc. (‘Company Name’ or ‘Company’) recommends reading the following terms and conditions, carefully: | ||
| - | ||
| -Workin' hard | ||
| -Workin' hard | ||
| - | ||
| -'70s are my favorite numbers,' she said. | ||
| -‘70s are my favorite numbers,’ she said. | ||
| - | ||
| -'70s fashion was weird. | ||
| -'70s fashion was weird. | ||
| - | ||
| -12\" record, 5'10\" height | ||
| -12″ record, 5′10″ height | ||
| - | ||
| -Model \"T2000\" | ||
| -Model “T2000” | ||
| - | ||
| -iPad 3's battery life is not great. | ||
| -iPad 3's battery life is not great. | ||
| - | ||
| -Book 'em, Danno. Rock 'n' roll. 'Cause 'twas the season. | ||
| -Book 'em, Danno. Rock 'n' roll. 'Cause 'twas the season. | ||
| - | ||
| -'85 was a good year. (The entire '80s were.) | ||
| -'85 was a good year. (The entire '80s were.) | ||
| - | ||
src/test/java/com/keenwrite/quotes/TypographerTest.java
| +/* Copyright 2021 White Magic Software, Ltd. -- All rights reserved. */ | ||
| +package com.keenwrite.quotes; | ||
| + | ||
| +import com.keenwrite.dom.DocumentParser; | ||
| +import org.junit.jupiter.api.Test; | ||
| + | ||
| +import java.io.BufferedReader; | ||
| +import java.io.IOException; | ||
| +import java.io.InputStreamReader; | ||
| + | ||
| +import static java.lang.System.lineSeparator; | ||
| +import static java.lang.System.out; | ||
| +import static java.util.stream.Collectors.joining; | ||
| +import static org.junit.jupiter.api.Assertions.assertNotNull; | ||
| + | ||
| +/** | ||
| + * Test to make sure that the {@link Typographer} can replace straight quotes | ||
| + * without affecting preformatted HTML text elements. | ||
| + */ | ||
| +class TypographerTest { | ||
| + | ||
| + private static final String LINE_SEP = lineSeparator(); | ||
| + | ||
| + @Test | ||
| + void test_Quotes_Straight_Curly() throws IOException { | ||
| + final var xhtml = read( "17165.html" ); | ||
| + final var doc = DocumentParser.parse( xhtml ); | ||
| + | ||
| + Typographer.curl( doc ); | ||
| + | ||
| + out.println( doc ); | ||
| + } | ||
| + | ||
| + /** | ||
| + * Opens a text file for reading. Callers are responsible for closing. | ||
| + * | ||
| + * @param filename The file to open. | ||
| + * @return An instance of {@link BufferedReader} that can be used to | ||
| + * read all the lines in the file. | ||
| + */ | ||
| + private BufferedReader open( final String filename ) { | ||
| + final var is = getClass().getResourceAsStream( filename ); | ||
| + assertNotNull( is ); | ||
| + | ||
| + return new BufferedReader( new InputStreamReader( is ) ); | ||
| + } | ||
| + | ||
| + @SuppressWarnings( "SameParameterValue" ) | ||
| + private String read( final String filename ) throws IOException { | ||
| + try( final var reader = open( filename ) ) { | ||
| + return reader.lines().collect( joining( LINE_SEP ) ); | ||
| + } | ||
| + } | ||
| +} | ||
src/main/java/com/keenwrite/processors/XhtmlProcessor.java
| package com.keenwrite.processors; | ||
| +import com.keenwrite.dom.DocumentParser; | ||
| import com.keenwrite.preferences.Key; | ||
| import com.keenwrite.preferences.Workspace; | ||
| import com.keenwrite.ui.heuristics.WordCounter; | ||
| import javafx.beans.property.StringProperty; | ||
| -import org.jsoup.nodes.Document; | ||
| +import org.w3c.dom.Document; | ||
| -import javax.xml.parsers.DocumentBuilderFactory; | ||
| -import javax.xml.transform.TransformerFactory; | ||
| -import javax.xml.transform.dom.DOMSource; | ||
| -import javax.xml.transform.stream.StreamResult; | ||
| import java.io.FileNotFoundException; | ||
| import java.nio.file.Path; | ||
| import java.util.Locale; | ||
| import java.util.Map; | ||
| -import java.util.Map.Entry; | ||
| import java.util.regex.Pattern; | ||
| import static com.keenwrite.Bootstrap.APP_TITLE_LOWERCASE; | ||
| +import static com.keenwrite.dom.DocumentParser.createMeta; | ||
| +import static com.keenwrite.dom.DocumentParser.walk; | ||
| import static com.keenwrite.events.StatusEvent.clue; | ||
| import static com.keenwrite.io.HttpFacade.httpGet; | ||
 | ||
| import static java.util.regex.Pattern.UNICODE_CHARACTER_CLASS; | ||
| import static java.util.regex.Pattern.compile; | ||
| -import static javax.xml.transform.OutputKeys.INDENT; | ||
| -import static javax.xml.transform.OutputKeys.OMIT_XML_DECLARATION; | ||
| -import static org.jsoup.Jsoup.parse; | ||
| -import static org.jsoup.nodes.Document.OutputSettings.Syntax; | ||
| +import static org.w3c.dom.Node.TEXT_NODE; | ||
| /** | ||
| - * Responsible for making an HTML document complete by wrapping it with html | ||
| + * Responsible for making an XHTML document complete by wrapping it with html | ||
| * and body elements. This doesn't have to be super-efficient because it's | ||
| * not run in real-time. | ||
| ||
| clue( "Main.status.typeset.xhtml" ); | ||
| - final var doc = parse( html ); | ||
| + final var decorated = | ||
| + "<html><head><title>untitled</title></head><body>" + | ||
| + html + | ||
| + "</body></html>"; | ||
| + final var doc = DocumentParser.parse( decorated ); | ||
| setMetaData( doc ); | ||
| - doc.outputSettings().syntax( Syntax.xml ); | ||
| - for( final var img : doc.getElementsByTag( "img" ) ) { | ||
| + walk( doc, "img", node -> { | ||
| try { | ||
| - final var imageFile = exportImage( img.attr( "src" ) ); | ||
| + final var attrs = node.getAttributes(); | ||
| - img.attr( "src", imageFile.toString() ); | ||
| + if( attrs != null ) { | ||
| + final var attr = attrs.getNamedItem( "src" ); | ||
| + | ||
| + if( attr != null ) { | ||
| + final var imageFile = exportImage( attr.getTextContent() ); | ||
| + | ||
| + attr.setTextContent( imageFile.toString() ); | ||
| + } | ||
| + } | ||
| } catch( final Exception ex ) { | ||
| clue( ex ); | ||
| } | ||
| - } | ||
| + } ); | ||
| - return doc.html(); | ||
| + //Typographer.curl( doc ); | ||
| + | ||
| + return DocumentParser.toString( doc ); | ||
| } | ||
| /** | ||
| * Applies the metadata fields to the document. | ||
| * | ||
| * @param doc The document to adorn with metadata. | ||
| */ | ||
| private void setMetaData( final Document doc ) { | ||
| - doc.title( getTitle() ); | ||
| - | ||
| final var metadata = createMetaData( doc ); | ||
| - final var head = doc.head(); | ||
| - metadata.entrySet().forEach( entry -> head.append( createMeta( entry ) ) ); | ||
| - } | ||
| - private String createMeta( final Entry<String, String> entry ) { | ||
| - return format( | ||
| - "<meta name='%s' content='%s'>", entry.getKey(), entry.getValue() | ||
| + walk( doc, "title", node -> node.setTextContent( getTitle() ) ); | ||
| + walk( doc, "head", node -> | ||
| + metadata.entrySet() | ||
| + .forEach( entry -> node.appendChild( createMeta( doc, entry ) ) ) | ||
| ); | ||
| } | ||
| ||
| // Strip comments, superfluous whitespace, DOCTYPE, and XML declarations. | ||
| if( mediaType.isSvg() ) { | ||
| - sanitize( imageFile ); | ||
| + DocumentParser.sanitize( imageFile ); | ||
| } | ||
| } | ||
| ||
| return imageFile; | ||
| - } | ||
| - | ||
| - /** | ||
| - * Remove whitespace, comments, and XML/DOCTYPE declarations to make | ||
| - * processing work with ConTeXt. | ||
| - * | ||
| - * @param path The SVG file to process. | ||
| - * @throws Exception The file could not be processed. | ||
| - */ | ||
| - private void sanitize( final Path path ) | ||
| - throws Exception { | ||
| - final var file = path.toFile(); | ||
| - | ||
| - final var dbf = DocumentBuilderFactory.newInstance(); | ||
| - dbf.setIgnoringComments( true ); | ||
| - dbf.setIgnoringElementContentWhitespace( true ); | ||
| - | ||
| - final var db = dbf.newDocumentBuilder(); | ||
| - final var document = db.parse( file ); | ||
| - | ||
| - final var tf = TransformerFactory.newInstance(); | ||
| - final var transformer = tf.newTransformer(); | ||
| - | ||
| - final var source = new DOMSource( document ); | ||
| - final var result = new StreamResult( file ); | ||
| - transformer.setOutputProperty( OMIT_XML_DECLARATION, "yes" ); | ||
| - transformer.setOutputProperty( INDENT, "no" ); | ||
| - transformer.transform( source, result ); | ||
| } | ||
| ||
| private String getWordCount( final Document doc ) { | ||
| - final var text = doc.wholeText(); | ||
| - final var wordCounter = WordCounter.create( getLocale() ); | ||
| - return valueOf( wordCounter.countWords( text ) ); | ||
| + final var sb = new StringBuilder( 65536 * 10 ); | ||
| + | ||
| + walk( doc, "*", node -> { | ||
| + if( node.getNodeType() == TEXT_NODE && node.getTextContent() != null ) { | ||
| + sb.append( node.getTextContent() ); | ||
| + } | ||
| + } ); | ||
| + | ||
| + return valueOf( WordCounter.create( getLocale() ).count( sb.toString() ) ); | ||
| } | ||