p12tic · mmMike · Jul 24, 2015 · Jul 31, 2015 · Jul 31, 2015 · Jul 31, 2015
diff --git a/Makefile b/Makefile
@@ -24,9 +24,11 @@ datarootdir = $(prefix)/share
 docdir = $(datarootdir)/cppreference/doc
 bookdir = $(datarootdir)/devhelp/books
 
+qhelpgenerator = qhelpgenerator
+
 #Version
 
-VERSION=20141118
+VERSION=20150809
 
 #STANDARD RULES
 
@@ -39,6 +41,7 @@ DISTFILES=	\
 		ddg_parse_html.py		\
 		devhelp2qch.xsl			\
 		fix_devhelp-links.py	\
+		gen_chapter_index.py	\
 		index2autolinker.py	\
 		index2browser.py		\
 		index2ddg.py			\
@@ -47,12 +50,11 @@ DISTFILES=	\
 		index2search.py			\
 		index2highlight.py		\
 		index_transform.py		\
-		index-chapters-c.xml	\
-		index-chapters-cpp.xml	\
 		index-functions.README	\
 		index-functions-c.xml	\
 		index-functions-cpp.xml	\
 		link_map.py		\
+		merge_devhelp.py	\
 		preprocess.py			\
 		preprocess.xsl			\
 		preprocess-css.css		\
@@ -126,47 +128,47 @@ release: all
 #WORKER RULES
 doc_html: output/reference
 
-doc_devhelp: output/cppreference-doc-en-c.devhelp2 output/cppreference-doc-en-cpp.devhelp2
+doc_devhelp: output/cppreference-doc-en-c.devhelp2 output/cppreference-doc-en-cpp.devhelp2 output/complete.devhelp2
 
 doc_qch: output/cppreference-doc-en-cpp.qch
 
 doc_doxygen: output/cppreference-doxygen-web.tag.xml output/cppreference-doxygen-local.tag.xml
 
-#builds the title<->location map
-output/link-map.xml: output/reference
-	./build_link_map.py
-
 #build the .devhelp2 index
 output/cppreference-doc-en-c.devhelp2: 		\
-		output/reference 		\
-		output/link-map.xml
-	./index2devhelp.py $(docdir)/html index-chapters-c.xml  \
+		output/reference
+	./gen_chapter_index.py --i output/reference/en/c.html --o output/index-chapters-c.xml
+	./index2devhelp.py $(docdir)/html output/index-chapters-c.xml  \
 		"C Standard Library reference" "cppreference-doc-en-c" "c" \
-		index-functions-c.xml "output/devhelp-index-c.xml"
-	./fix_devhelp-links.py "output/devhelp-index-c.xml"  \
-		"output/cppreference-doc-en-c.devhelp2"
+		index-functions-c.xml "output/cppreference-doc-en-c.devhelp2"
+
 
 output/cppreference-doc-en-cpp.devhelp2:	\
-		output/reference 		\
-		output/link-map.xml
-	./index2devhelp.py $(docdir)/html index-chapters-cpp.xml  \
+		output/reference
+	./gen_chapter_index.py --i output/reference/en/cpp.html --o output/index-chapters-cpp.xml
+	./index2devhelp.py $(docdir)/html output/index-chapters-cpp.xml  \
 		"C++ Standard Library reference" "cppreference-doc-en-cpp" "cpp" \
-		index-functions-cpp.xml "output/devhelp-index-cpp.xml"
-	./fix_devhelp-links.py "output/devhelp-index-cpp.xml" \
-		"output/cppreference-doc-en-cpp.devhelp2"
+		index-functions-cpp.xml "output/cppreference-doc-en-cpp.devhelp2"
+
+output/complete.devhelp2:	\
+		output/reference output/cppreference-doc-en-c.devhelp2 output/cppreference-doc-en-cpp.devhelp2
+	./merge_devhelp.py \
+		--c "output/cppreference-doc-en-c.devhelp2" \
+		--cpp "output/cppreference-doc-en-cpp.devhelp2" \
+		--out "output/complete.devhelp2"
 
 #build the .qch (QT help) file
-output/cppreference-doc-en-cpp.qch: output/qch-help-project-cpp.xml
+output/cppreference-doc-en-cpp.qch: output/complete.xml
 	#qhelpgenerator only works if the project file is in the same directory as the documentation
-	cp "output/qch-help-project-cpp.xml" "output/reference/qch.xml"
+	cp "output/complete.xml" "output/reference/qch.xml"
 
 	pushd "output/reference" > /dev/null; \
-	qhelpgenerator "qch.xml" -o "../cppreference-doc-en-cpp.qch"; \
+	$(qhelpgenerator) "qch.xml" -o "../cppreference-doc-en-cpp.qch"; \
 	popd > /dev/null
 
 	rm -f "output/reference/qch.xml"
 
-output/qch-help-project-cpp.xml: output/cppreference-doc-en-cpp.devhelp2
+output/complete.xml: output/complete.devhelp2
 	#build the file list
 	echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?><files>" > "output/qch-files.xml"
 
@@ -178,20 +180,18 @@ output/qch-help-project-cpp.xml: output/cppreference-doc-en-cpp.devhelp2
 	echo "</files>" >> "output/qch-files.xml"
 
 	#create the project (copies the file list)
-	xsltproc devhelp2qch.xsl "output/cppreference-doc-en-cpp.devhelp2" > \
-		"output/qch-help-project-cpp.xml"
+	xsltproc devhelp2qch.xsl "output/complete.devhelp2" > \
+		"output/complete.xml"
 
 # build doxygen tag file
 output/cppreference-doxygen-local.tag.xml: 		\
-		output/reference 		\
-		output/link-map.xml
+		output/reference
 	./index2doxygen-tag.py "output/link-map.xml" \
 		"index-functions-cpp.xml" \
 		"output/cppreference-doxygen-local.tag.xml"
 
 output/cppreference-doxygen-web.tag.xml: 		\
-		output/reference 		\
-		output/link-map.xml
+		output/reference
 	./index2doxygen-tag.py web \
 		"index-functions-cpp.xml" \
 		"output/cppreference-doxygen-web.tag.xml"
@@ -230,5 +230,9 @@ source:
 	  --timeout=180 --no-verbose \
 	  --retry-connrefused --waitretry=1 --read-timeout=20 \
 	  http://en.cppreference.com/w/ ; \
+	wget --adjust-extension --page-requisites --convert-links \
+	  --timeout=180 --no-verbose \
+	  --retry-connrefused --waitretry=1 --read-timeout=20 \
+	  http://en.cppreference.com/w/Cppreference:FAQ ; \
 	popd > /dev/null
 
diff --git a/build_link_map.py b/build_link_map.py
@@ -28,7 +28,7 @@
 
 # returns a dict { title -> filename }.
 # directory - either 'output/reference' or 'reference'
-def build_link_map(directory):
+def build_link_map_impl(directory):
     # find all html files
     html_files = []
     for root, dirnames, filenames in os.walk(directory):
@@ -57,11 +57,14 @@ def build_link_map(directory):
         link_map.add_link(title, target)
     return link_map
 
-def main():
-    link_map = build_link_map('output/reference')
+def build_link_map():
+    link_map = build_link_map_impl('output/reference')
 
     # create an xml file containing mapping between page title and actual location
     link_map.write('output/link-map.xml')
 
+def main():
+    build_link_map()
+
 if __name__ == "__main__":
     main()
diff --git a/clean_faq.py b/clean_faq.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+#   Copyright (C) 2015  Michael Munzert <[email protected]>
+#
+#   This file is part of cppreference-doc
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see http://www.gnu.org/licenses/.
+
+import os
+import re
+import shutil
+import urllib.parse
+
+from bs4 import BeautifulSoup
+
+def urldecode(s):
+    return urllib.parse.unquote(s)
+
+def clean_faq(output_path):
+    # Fix FAQ links
+    shutil.move(os.path.join(output_path, 'reference/en/Cppreference:FAQ.html'), 
+                os.path.join(output_path, 'reference/en/FAQ.html'))
+    for fn in ['reference/en/c.html', 'reference/en/cpp.html']:
+        with open(os.path.join(output_path, fn), 'rb') as f:
+            soup = BeautifulSoup(f.read(), "lxml")
+
+        for link in soup.find_all('a'):
+            if link.text == 'FAQ':
+                link['href'] = 'FAQ.html'
+                break;
+
+        with open(os.path.join(output_path, fn), 'wb') as f:
+            f.write(soup.prettify('utf-8'))
+
+    # clean FAQ.html        
+    with open(os.path.join(output_path, 'reference/en/FAQ.html'), 'rb') as f:
+        soup = BeautifulSoup(f.read(), "lxml")
+
+    for link in soup.find_all('a'):
+        try:
+            href = link['href']
+            if 'FAQ.html' in href:
+                link['href'] = re.sub('([^#]*)', 'FAQ.html', href)
+        except KeyError:
+            pass
+
+    navbar_head = soup.find(attrs={'class': 't-navbar-head'})
+    navbar_head.a.string   = 'Index'
+    navbar_head.a['href']  = 'index.html'
+    navbar_head.a['title'] = 'index'
+    navbar_head.div.extract()
+
+    with open(os.path.join(output_path, 'reference/en/FAQ.html'), 'wb') as f:
+        f.write(soup.prettify('utf-8'))
+
diff --git a/devhelp2qch.xsl b/devhelp2qch.xsl
@@ -38,12 +38,18 @@
     </virtualFolder>
 
     <customFilter name="{@title}">
+      <filterAttribute>
+        <xsl:value-of select="$qch-namespace" />
+      </filterAttribute>
       <filterAttribute>
         <xsl:value-of select="@name" />
       </filterAttribute>
     </customFilter>
 
     <filterSection>
+      <filterAttribute>
+        <xsl:value-of select="$qch-namespace" />
+      </filterAttribute>
       <filterAttribute>
         <xsl:value-of select="@name" />
       </filterAttribute>
@@ -77,7 +83,8 @@
     <keyword name="{@name}" id="{@name}" ref="{@link}" />
     <!-- Add an additional id for libc++ users -->
     <xsl:if test="starts-with(@name, 'std::')">
-      <keyword name="{@name}" id="{concat('std::___LIBCPP_ABI_VERSION::', substring(@name, 6))}" ref="{@link}" />
+      <keyword id="{concat('std::__LIBCPP_ABI_VERSION::', substring(@name, 6))}" ref="{@link}" />
+      <keyword id="{concat('std::__1::', substring(@name, 6))}" ref="{@link}" />
     </xsl:if>
   </xsl:if>
 </xsl:template>

diff --git a/gen_chapter_index.py b/gen_chapter_index.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+#   Copyright (C) 2015  Michael Munzert <[email protected]>
+#
+#   This file is part of cppreference-doc
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see http://www.gnu.org/licenses/.
+
+# Generate the chapter index
+
+import argparse
+from lxml import etree
+from bs4 import BeautifulSoup
+import re
+
+def main():
+    parser = argparse.ArgumentParser(description='Generate the chapter index')
+    parser.add_argument('--i', help='source file', required=True)
+    parser.add_argument('--o', help='output file', required=True)
+
+    args = parser.parse_args()
+
+    with open(args.i, 'r') as f:
+        soup = BeautifulSoup(f.read(), "lxml")
+
+    root = etree.XML('''<chapters xmlns="http://www.devhelp.net/book"/>''')
+    for p in soup.find_all('p'):
+        for tag in p:
+            if tag.name == "b":
+                try:
+                    last = etree.SubElement(root, "sub")
+                    last.attrib["name"] = re.sub(r"\s+", ' ', tag.a.text.strip())
+                    last.attrib["link"] = "en/" + tag.a["href"]
+                except KeyError:
+                    pass
+            if tag.name == "a":
+                try:
+                    child = etree.SubElement(last, "sub")
+                    child.attrib["name"] = re.sub(r"\s+", ' ', tag.text.strip())
+                    child.attrib["link"] = "en/" + tag["href"]
+                except KeyError:
+                    pass
+
+    etree.ElementTree(root).write(args.o, pretty_print=True, encoding='utf-8')
+
+
+if __name__ == "__main__":
+    main()
diff --git a/index-chapters-c.xml b/index-chapters-c.xml