From d4c456bdef566128ed18bcf066b430247ddb60b1 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Tue, 7 Feb 2017 20:34:49 -0800 Subject: Add invalid type assertion for offset_debug This will make sure we don't forget to implement offset_debug for new node types if they ever happen (really it's mostly for consistency). --- src/pugixml.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pugixml.cpp b/src/pugixml.cpp index 1a69ac6..7368184 100644 --- a/src/pugixml.cpp +++ b/src/pugixml.cpp @@ -6286,6 +6286,7 @@ namespace pugi return _root->value && (_root->header & impl::xml_memory_page_value_allocated_or_shared_mask) == 0 ? _root->value - doc.buffer : -1; default: + assert(false && "Invalid node type"); return -1; } } -- cgit v1.2.3 From ba39838ab587173ed6f08841f4a135403d605911 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Wed, 8 Feb 2017 00:09:32 -0800 Subject: tests: Add more XPath out of memory tests --- tests/test_xpath.cpp | 13 +++++++++++++ tests/test_xpath_variables.cpp | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/tests/test_xpath.cpp b/tests/test_xpath.cpp index 6cae607..a2af5e6 100644 --- a/tests/test_xpath.cpp +++ b/tests/test_xpath.cpp @@ -521,6 +521,19 @@ TEST(xpath_memory_concat_massive) CHECK(size == 5001); } +TEST_XML(xpath_memory_translate_table, "a") +{ + test_runner::_memory_fail_threshold = 32768 + 4096 + 128; + + // 128b per table => we need 32 translate calls to exhaust a page + std::basic_string query = STR("concat("); + for (int i = 0; i < 32; ++i) + query += STR("translate(.,'a','A'),"); + query += STR("'')"); + + CHECK_ALLOC_FAIL(pugi::xpath_query(query.c_str()).evaluate_string(doc.first_child()) == STR("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); +} + TEST_XML(xpath_sort_copy_share, "test") { // copy sharing shares the name/value data for nodes that can potentially make document order optimization invalid (silently) diff --git a/tests/test_xpath_variables.cpp b/tests/test_xpath_variables.cpp index 2033a8e..c64e0e6 100644 --- a/tests/test_xpath_variables.cpp +++ b/tests/test_xpath_variables.cpp @@ -474,6 +474,18 @@ TEST_XML(xpath_variables_copy_out_of_memory, "") CHECK(set2.get(STR("d"))->get_node_set().size() == 2); } +TEST(xpath_variables_copy_out_of_memory_clone) +{ + xpath_variable_set set1; + set1.set(STR("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"), true); + + xpath_variable_set set2 = set1; + + test_runner::_memory_fail_threshold = 60; + + CHECK_ALLOC_FAIL(xpath_variable_set set3 = set1); +} + #ifdef PUGIXML_HAS_MOVE TEST_XML(xpath_variables_move, "") { -- cgit v1.2.3 From 4bab082a27daa050aa7d704ffdc1b4ee9a3a4c4e Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Wed, 8 Feb 2017 00:31:05 -0800 Subject: tests: Fix clang build --- tests/test_xpath.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_xpath.cpp b/tests/test_xpath.cpp index a2af5e6..417440b 100644 --- a/tests/test_xpath.cpp +++ b/tests/test_xpath.cpp @@ -531,7 +531,7 @@ TEST_XML(xpath_memory_translate_table, "a") query += STR("translate(.,'a','A'),"); query += STR("'')"); - CHECK_ALLOC_FAIL(pugi::xpath_query(query.c_str()).evaluate_string(doc.first_child()) == STR("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + CHECK_ALLOC_FAIL(CHECK(!pugi::xpath_query(query.c_str()))); } TEST_XML(xpath_sort_copy_share, "test") -- cgit v1.2.3 From e748f435e5481b5a44686486e8f467823688b2c0 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Wed, 8 Feb 2017 01:18:11 -0800 Subject: tests: Increase the number of translate calls This should make the test fail on a 32-bit target. --- tests/test_xpath.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_xpath.cpp b/tests/test_xpath.cpp index 417440b..69513fc 100644 --- a/tests/test_xpath.cpp +++ b/tests/test_xpath.cpp @@ -525,9 +525,9 @@ TEST_XML(xpath_memory_translate_table, "a") { test_runner::_memory_fail_threshold = 32768 + 4096 + 128; - // 128b per table => we need 32 translate calls to exhaust a page + // 128b per table => we need 32+ translate calls to exhaust a page std::basic_string query = STR("concat("); - for (int i = 0; i < 32; ++i) + for (int i = 0; i < 64; ++i) query += STR("translate(.,'a','A'),"); query += STR("'')"); -- cgit v1.2.3 From 00ef791078ec318f663b0cffdb18fd928394d591 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Tue, 7 Feb 2017 23:34:39 -0800 Subject: fuzz: Use libFuzzer instead of afl-fuzz This allows us to have faster fuzz cycles since the fuzzer is in-process. --- Makefile | 11 +++++++---- tests/fuzz_parse.cpp | 18 ++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index f9b26d6..e483e02 100644 --- a/Makefile +++ b/Makefile @@ -68,10 +68,9 @@ test: $(EXECUTABLE) ./$(EXECUTABLE) endif -fuzz: - @mkdir -p $(BUILD) - $(AFL)/afl-clang++ tests/fuzz_parse.cpp tests/allocator.cpp src/pugixml.cpp $(CXXFLAGS) -o $(BUILD)/fuzz_parse - $(AFL)/afl-fuzz -i tests/data_fuzz_parse -o $(BUILD)/fuzz_parse_out -x $(AFL)/testcases/_extras/xml/ -- $(BUILD)/fuzz_parse @@ +fuzz_%: $(BUILD)/fuzz_% + @mkdir -p build/$@ + $< build/$@ tests/data_$* clean: rm -rf $(BUILD) @@ -87,6 +86,10 @@ build/pugixml-%: .FORCE | $(RELEASE) $(EXECUTABLE): $(OBJECTS) $(CXX) $(OBJECTS) $(LDFLAGS) -o $@ +$(BUILD)/fuzz_%: tests/fuzz_%.cpp src/pugixml.cpp + @mkdir -p $(BUILD) + clang++ $(CXXFLAGS) -fsanitize=address -fsanitize-coverage=trace-pc-guard $^ libFuzzer.a -o $@ + $(BUILD)/%.o: % @mkdir -p $(dir $@) $(CXX) $< $(CXXFLAGS) -c -MMD -MP -o $@ diff --git a/tests/fuzz_parse.cpp b/tests/fuzz_parse.cpp index e758196..94c610a 100644 --- a/tests/fuzz_parse.cpp +++ b/tests/fuzz_parse.cpp @@ -1,16 +1,14 @@ #include "../src/pugixml.hpp" -#include "allocator.hpp" -int main(int argc, const char** argv) -{ - pugi::set_memory_management_functions(memory_allocate, memory_deallocate); +#include +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) +{ pugi::xml_document doc; - for (int i = 1; i < argc; ++i) - { - doc.load_file(argv[i]); - doc.load_file(argv[i], pugi::parse_minimal); - doc.load_file(argv[i], pugi::parse_full); - } + doc.load_buffer(Data, Size); + doc.load_buffer(Data, Size, pugi::parse_minimal); + doc.load_buffer(Data, Size, pugi::parse_full); + + return 0; } -- cgit v1.2.3 From 8b15ae801531f64804566391bfbd10db451956ca Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Wed, 8 Feb 2017 08:48:33 -0800 Subject: tests: Add a script to set up fuzzing tools This downloads a clang build that has support for instrumentation, and also downloads and compiles libFuzzer.a. --- tests/fuzz_setup.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/fuzz_setup.sh diff --git a/tests/fuzz_setup.sh b/tests/fuzz_setup.sh new file mode 100644 index 0000000..df5462c --- /dev/null +++ b/tests/fuzz_setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +sudo apt-get --yes install subversion screen gcc g++ cmake ninja-build golang autoconf libtool apache2 python-dev pkg-config zlib1g-dev libgcrypt11-dev + +git clone https://chromium.googlesource.com/chromium/src/tools/clang +clang/scripts/update.py +sudo cp -rf third_party/llvm-build/Release+Asserts/lib/* /usr/local/lib/ +sudo cp -rf third_party/llvm-build/Release+Asserts/bin/* /usr/local/bin + +svn co http://llvm.org/svn/llvm-project/llvm/trunk/lib/Fuzzer +Fuzzer/build.sh -- cgit v1.2.3 From 8c62fa9121eacf5cde1ed59bf0cca896227a44d5 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Wed, 8 Feb 2017 08:48:54 -0800 Subject: tests: Add XPath fuzzing Only fuzz the parser for now. --- tests/data_fuzz_xpath/basic.xpath | 1 + tests/data_fuzz_xpath/functions.xpath | 1 + tests/data_fuzz_xpath/math.xpath | 1 + tests/data_fuzz_xpath/path.xpath | 1 + tests/data_fuzz_xpath/predicate.xpath | 1 + tests/fuzz_xpath.cpp | 26 ++++++++++++++++++++++++++ 6 files changed, 31 insertions(+) create mode 100644 tests/data_fuzz_xpath/basic.xpath create mode 100644 tests/data_fuzz_xpath/functions.xpath create mode 100644 tests/data_fuzz_xpath/math.xpath create mode 100644 tests/data_fuzz_xpath/path.xpath create mode 100644 tests/data_fuzz_xpath/predicate.xpath create mode 100644 tests/fuzz_xpath.cpp diff --git a/tests/data_fuzz_xpath/basic.xpath b/tests/data_fuzz_xpath/basic.xpath new file mode 100644 index 0000000..ccbaf23 --- /dev/null +++ b/tests/data_fuzz_xpath/basic.xpath @@ -0,0 +1 @@ +a/b/c \ No newline at end of file diff --git a/tests/data_fuzz_xpath/functions.xpath b/tests/data_fuzz_xpath/functions.xpath new file mode 100644 index 0000000..ec24b4f --- /dev/null +++ b/tests/data_fuzz_xpath/functions.xpath @@ -0,0 +1 @@ +sum(nodes) + round(concat(//a[translate(@id, 'abc', '012')])) diff --git a/tests/data_fuzz_xpath/math.xpath b/tests/data_fuzz_xpath/math.xpath new file mode 100644 index 0000000..7f6e968 --- /dev/null +++ b/tests/data_fuzz_xpath/math.xpath @@ -0,0 +1 @@ +1+2*3 div 4 mod 5-6 \ No newline at end of file diff --git a/tests/data_fuzz_xpath/path.xpath b/tests/data_fuzz_xpath/path.xpath new file mode 100644 index 0000000..82cace9 --- /dev/null +++ b/tests/data_fuzz_xpath/path.xpath @@ -0,0 +1 @@ +@*/ancestor::*/near-north/*[4]/@*/preceding::text() \ No newline at end of file diff --git a/tests/data_fuzz_xpath/predicate.xpath b/tests/data_fuzz_xpath/predicate.xpath new file mode 100644 index 0000000..7161d55 --- /dev/null +++ b/tests/data_fuzz_xpath/predicate.xpath @@ -0,0 +1 @@ +library/nodes[@id=12]/element[@type='translate'][1] \ No newline at end of file diff --git a/tests/fuzz_xpath.cpp b/tests/fuzz_xpath.cpp new file mode 100644 index 0000000..c7ff4cd --- /dev/null +++ b/tests/fuzz_xpath.cpp @@ -0,0 +1,26 @@ +#include "../src/pugixml.hpp" + +#include +#include + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) +{ + char* text = new char[Size + 1]; + memcpy(text, Data, Size); + text[Size] = 0; + +#ifdef PUGIXML_NO_EXCEPTIONS + pugi::xpath_query q(text); +#else + try + { + pugi::xpath_query q(text); + } + catch (pugi::xpath_exception&) + { + } +#endif + + delete[] text; + return 0; +} -- cgit v1.2.3 From ea544eb48bb7414a76b059ff285b63ce1cf1fbda Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Thu, 9 Feb 2017 09:53:04 -0800 Subject: tests: Add fuzzing dictionaries Hopefully this will allow for better fuzzing coverage --- Makefile | 2 +- tests/fuzz_parse.dict | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++ tests/fuzz_xpath.dict | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 tests/fuzz_parse.dict create mode 100644 tests/fuzz_xpath.dict diff --git a/Makefile b/Makefile index e483e02..500675c 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ endif fuzz_%: $(BUILD)/fuzz_% @mkdir -p build/$@ - $< build/$@ tests/data_$* + $< build/$@ tests/data_$* -max_len=1024 -dict=tests/fuzz_$*.dict clean: rm -rf $(BUILD) diff --git a/tests/fuzz_parse.dict b/tests/fuzz_parse.dict new file mode 100644 index 0000000..b1a0067 --- /dev/null +++ b/tests/fuzz_parse.dict @@ -0,0 +1,72 @@ +# +# AFL dictionary for XML +# ---------------------- +# +# Several basic syntax elements and attributes, modeled on libxml2. +# +# Created by Michal Zalewski +# + +attr_encoding=" encoding=\"1\"" +attr_generic=" a=\"1\"" +attr_href=" href=\"1\"" +attr_standalone=" standalone=\"no\"" +attr_version=" version=\"1\"" +attr_xml_base=" xml:base=\"1\"" +attr_xml_id=" xml:id=\"1\"" +attr_xml_lang=" xml:lang=\"1\"" +attr_xml_space=" xml:space=\"1\"" +attr_xmlns=" xmlns=\"1\"" + +entity_builtin="<" +entity_decimal="" +entity_external="&a;" +entity_hex="" + +string_any="ANY" +string_brackets="[]" +string_cdata="CDATA" +string_col_fallback=":fallback" +string_col_generic=":a" +string_col_include=":include" +string_dashes="--" +string_empty="EMPTY" +string_empty_dblquotes="\"\"" +string_empty_quotes="''" +string_entities="ENTITIES" +string_entity="ENTITY" +string_fixed="#FIXED" +string_id="ID" +string_idref="IDREF" +string_idrefs="IDREFS" +string_implied="#IMPLIED" +string_nmtoken="NMTOKEN" +string_nmtokens="NMTOKENS" +string_notation="NOTATION" +string_parentheses="()" +string_pcdata="#PCDATA" +string_percent="%a" +string_public="PUBLIC" +string_required="#REQUIRED" +string_schema=":schema" +string_system="SYSTEM" +string_ucs4="UCS-4" +string_utf16="UTF-16" +string_utf8="UTF-8" +string_xmlns="xmlns:" + +tag_attlist="" +tag_doctype="" +tag_open_close="" +tag_open_exclamation="" +tag_xml_q="" diff --git a/tests/fuzz_xpath.dict b/tests/fuzz_xpath.dict new file mode 100644 index 0000000..c469f6e --- /dev/null +++ b/tests/fuzz_xpath.dict @@ -0,0 +1,72 @@ +"boolean" +"count" +"contains" +"concat" +"ceiling" +"false" +"floor" +"id" +"last" +"lang" +"local-name" +"name" +"namespace-uri" +"normalize-space" +"not" +"number" +"position" +"round" +"string" +"string-length" +"starts-with" +"substring-before" +"substring-after" +"substring" +"sum" +"translate" +"true" +"ancestor" +"ancestor-or-self" +"attribute" +"child" +"descendant" +"descendant-or-self" +"following" +"following-sibling" +"namespace" +"parent" +"preceding" +"preceding-sibling" +"self" +"comment" +"node" +"processing-instruction" +"text" +"or" +"and" +"div" +"mod" +">" +">=" +"<" +"<=" +"!" +"!=" +"=" +"+" +"-" +"*" +"|" +"$" +"(" +")" +"[" +"]" +"," +"//" +"/" +".." +"." +"@" +"::" +":" -- cgit v1.2.3 From ec984370fb525d0cbc20b009f69c9c5eaec022a7 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Fri, 10 Feb 2017 23:51:47 -0800 Subject: tests: Fix fuzz_setup.sh Make the file executable, fix Windows newlines and fix clang setup. --- tests/fuzz_setup.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) mode change 100644 => 100755 tests/fuzz_setup.sh diff --git a/tests/fuzz_setup.sh b/tests/fuzz_setup.sh old mode 100644 new mode 100755 index df5462c..05a40cb --- a/tests/fuzz_setup.sh +++ b/tests/fuzz_setup.sh @@ -1,11 +1,13 @@ -#!/bin/bash - -sudo apt-get --yes install subversion screen gcc g++ cmake ninja-build golang autoconf libtool apache2 python-dev pkg-config zlib1g-dev libgcrypt11-dev - -git clone https://chromium.googlesource.com/chromium/src/tools/clang -clang/scripts/update.py -sudo cp -rf third_party/llvm-build/Release+Asserts/lib/* /usr/local/lib/ -sudo cp -rf third_party/llvm-build/Release+Asserts/bin/* /usr/local/bin - -svn co http://llvm.org/svn/llvm-project/llvm/trunk/lib/Fuzzer -Fuzzer/build.sh +#!/bin/bash + +sudo apt-get --yes install subversion screen gcc g++ cmake ninja-build golang autoconf libtool apache2 python-dev pkg-config zlib1g-dev libgcrypt11-dev + +cd clang +git clone https://chromium.googlesource.com/chromium/src/tools/clang +cd .. +clang/clang/scripts/update.py +sudo cp -rf third_party/llvm-build/Release+Asserts/lib/* /usr/local/lib/ +sudo cp -rf third_party/llvm-build/Release+Asserts/bin/* /usr/local/bin + +svn co http://llvm.org/svn/llvm-project/llvm/trunk/lib/Fuzzer +Fuzzer/build.sh -- cgit v1.2.3