lexer: Make a lexer for our config file format

This lexer will be used in our PKCS#11 persistence format as well.

https://bugs.freedesktop.org/show_bug.cgi?id=62156

5 files changed, 605 insertions, 0 deletions

diff --git a/common/Makefile.am b/common/Makefile.am
index d914fa8..3522acb 100644
--- a/common/Makefile.am
+++ b/common/Makefile.am
@@ -25,6 +25,7 @@ libp11_library_la_SOURCES = \
 	constants.c constants.h \
 	debug.c debug.h \
 	dict.c dict.h \
+	lexer.c lexer.h \
 	library.c library.h \
 	pkcs11.h pkcs11x.h \
 	$(NULL)
diff --git a/common/lexer.c b/common/lexer.c
new file mode 100644
index 0000000..9898e2c
--- /dev/null
+++ b/common/lexer.c
@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) 2005 Stefan Walter
+ * Copyright (c) 2011 Collabora Ltd.
+ * Copyright (c) 2013 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *     * Redistributions of source code must retain the above
+ *       copyright notice, this list of conditions and the
+ *       following disclaimer.
+ *     * Redistributions in binary form must reproduce the
+ *       above copyright notice, this list of conditions and
+ *       the following disclaimer in the documentation and/or
+ *       other materials provided with the distribution.
+ *     * The names of contributors to this software may not be
+ *       used to endorse or promote products derived from this
+ *       software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ *
+ * CONTRIBUTORS
+ *  Stef Walter <stefw@redhat.com>
+ */
+
+#include "config.h"
+
+#define P11_DEBUG_FLAG P11_DEBUG_CONF
+#include "debug.h"
+#include "lexer.h"
+#include "library.h"
+
+#include <assert.h>
+#include <ctype.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+void
+p11_lexer_init (p11_lexer *lexer,
+                const char *filename,
+                const char *data,
+                size_t length)
+{
+	return_if_fail (lexer != NULL);
+
+	memset (lexer, 0, sizeof (p11_lexer));
+	lexer->at = data;
+	lexer->remaining = length;
+
+	return_if_fail (filename != NULL);
+	lexer->filename = strdup (filename);
+	return_if_fail (lexer->filename != NULL);
+}
+
+static void
+clear_state (p11_lexer *lexer)
+{
+	switch (lexer->tok_type) {
+	case TOK_FIELD:
+		free (lexer->tok.field.name);
+		free (lexer->tok.field.value);
+		break;
+	case TOK_SECTION:
+		free (lexer->tok.section.name);
+		break;
+	case TOK_PEM:
+	case TOK_EOF:
+		break;
+	}
+
+	memset (&lexer->tok, 0, sizeof (lexer->tok));
+	lexer->tok_type = TOK_EOF;
+	lexer->complained = false;
+}
+
+bool
+p11_lexer_next (p11_lexer *lexer,
+                bool *failed)
+{
+	const char *colon;
+	const char *value;
+	const char *line;
+	const char *end;
+	const char *pos;
+	char *part;
+
+	return_val_if_fail (lexer != NULL, false);
+
+	clear_state (lexer);
+	*failed = false;
+
+	/* Go through lines and process them */
+	while (lexer->remaining != 0) {
+		assert (lexer->remaining > 0);
+
+		/* Is this line the start of a PEM block? */
+		if (strncmp (lexer->at, "-----BEGIN ", 11) == 0) {
+			pos = strnstr (lexer->at, "\n-----END ", lexer->remaining);
+			if (pos != NULL) {
+				end = memchr (pos + 1, '\n', lexer->remaining - (pos - lexer->at) - 1);
+				if (end)
+					end += 1;
+				else
+					end = lexer->at + lexer->remaining;
+				lexer->tok_type = TOK_PEM;
+				lexer->tok.pem.begin = lexer->at;
+				lexer->tok.pem.length = end - lexer->at;
+				assert (end - lexer->at <= lexer->remaining);
+				lexer->remaining -= (end - lexer->at);
+				lexer->at = end;
+				return true;
+			}
+
+			p11_lexer_msg (lexer, "invalid pem block: no ending line");
+			if (failed)
+				*failed = true;
+			return false;
+		}
+
+		line = lexer->at;
+		end = memchr (lexer->at, '\n', lexer->remaining);
+		if (end == NULL) {
+			end = lexer->at + lexer->remaining;
+			lexer->remaining = 0;
+			lexer->at = end;
+		} else {
+			assert ((end - lexer->at) + 1 <= lexer->remaining);
+			lexer->remaining -= (end - lexer->at) + 1;
+			lexer->at = end + 1;
+		}
+
+		/* Strip whitespace from line */
+		while (line != end && isspace (line[0]))
+			++line;
+		while (line != end && isspace (*(end - 1)))
+			--end;
+
+		/* Empty lines / comments at start */
+		if (line == end || line[0] == '#')
+			continue;
+
+		/* Is the the a section ? */
+		if (line[0] == '[') {
+			if (*(end - 1) != ']') {
+				part = strndup (line, end - line);
+				p11_lexer_msg (lexer, "invalid section header: missing braces");
+				free (part);
+				if (failed)
+					*failed = true;
+				return false;
+			}
+
+			lexer->tok_type = TOK_SECTION;
+			lexer->tok.section.name = strndup (line + 1, (end - line) - 2);
+			return_val_if_fail (lexer->tok.section.name != NULL, false);
+			return true;
+		}
+
+		/* Look for the break between name: value on the same line */
+		colon = memchr (line, ':', end - line);
+		if (!colon) {
+			part = strndup (line, end - line);
+			p11_lexer_msg (lexer, "invalid field line: no colon");
+			free (part);
+			if (failed)
+				*failed = true;
+			return false;
+		}
+
+		/* Strip whitespace from name and value */
+		value = colon + 1;
+		while (value != end && isspace (value[0]))
+			++value;
+		while (line != colon && isspace (*(colon - 1)))
+			--colon;
+
+		lexer->tok_type = TOK_FIELD;
+		lexer->tok.field.name = strndup (line, colon - line);
+		lexer->tok.field.value = strndup (value, end - value);
+		return_val_if_fail (lexer->tok.field.name && lexer->tok.field.value, false);
+		return true;
+	}
+
+	return false;
+}
+
+void
+p11_lexer_done (p11_lexer *lexer)
+{
+	return_if_fail (lexer != NULL);
+	clear_state (lexer);
+	free (lexer->filename);
+	memset (lexer, 0, sizeof (p11_lexer));
+}
+
+void
+p11_lexer_msg (p11_lexer *lexer,
+               const char *msg)
+{
+	return_if_fail (lexer != NULL);
+
+	if (lexer->complained)
+		return;
+
+	switch (lexer->tok_type) {
+	case TOK_FIELD:
+		p11_message ("%s: %s: %s", lexer->filename,
+		             lexer->tok.field.name, msg);
+		break;
+	case TOK_SECTION:
+		p11_message ("%s: [%s]: %s", lexer->filename,
+		             lexer->tok.section.name, msg);
+		break;
+	case TOK_PEM:
+		p11_message ("%s: BEGIN ...: %s", lexer->filename, msg);
+		break;
+	default:
+		p11_message ("%s: %s", lexer->filename, msg);
+		break;
+	}
+
+	lexer->complained = true;
+}
diff --git a/common/lexer.h b/common/lexer.h
new file mode 100644
index 0000000..9daf296
--- /dev/null
+++ b/common/lexer.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2005 Stefan Walter
+ * Copyright (c) 2011 Collabora Ltd.
+ * Copyright (c) 2013 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *     * Redistributions of source code must retain the above
+ *       copyright notice, this list of conditions and the
+ *       following disclaimer.
+ *     * Redistributions in binary form must reproduce the
+ *       above copyright notice, this list of conditions and
+ *       the following disclaimer in the documentation and/or
+ *       other materials provided with the distribution.
+ *     * The names of contributors to this software may not be
+ *       used to endorse or promote products derived from this
+ *       software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ * Author: Stef Walter <stefw@redhat.com>
+ */
+
+#ifndef P11_LEXER_H__
+#define P11_LEXER_H__
+
+#include "compat.h"
+
+enum {
+	TOK_EOF = 0,
+	TOK_SECTION = 1,
+	TOK_FIELD,
+	TOK_PEM,
+};
+
+typedef struct {
+	char *filename;
+	const char *at;
+	int remaining;
+	int complained;
+
+	int tok_type;
+	union {
+		struct {
+			char *name;
+		} section;
+		struct {
+			char *name;
+			char *value;
+		} field;
+		struct {
+			const char *begin;
+			size_t length;
+		} pem;
+	} tok;
+} p11_lexer;
+
+void             p11_lexer_init               (p11_lexer *lexer,
+                                               const char *filename,
+                                               const char *data,
+                                               size_t length);
+
+bool             p11_lexer_next               (p11_lexer *lexer,
+                                               bool *failed);
+
+void             p11_lexer_done               (p11_lexer *lexer);
+
+void             p11_lexer_msg                (p11_lexer *lexer,
+                                               const char *msg);
+
+#endif /* P11_LEXER_H__ */
diff --git a/common/tests/Makefile.am b/common/tests/Makefile.am
index 582c1fb..f31cebb 100644
--- a/common/tests/Makefile.am
+++ b/common/tests/Makefile.am
@@ -20,6 +20,7 @@ CHECK_PROGS = \
 	test-constants \
 	test-attrs \
 	test-buffer \
+	test-lexer \
 	$(NULL)
 
 noinst_PROGRAMS = \
diff --git a/common/tests/test-lexer.c b/common/tests/test-lexer.c
new file mode 100644
index 0000000..02ea5c5
--- /dev/null
+++ b/common/tests/test-lexer.c
@@ -0,0 +1,281 @@
+/*
+ * Copyright (c) 2013 Red Hat Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *     * Redistributions of source code must retain the above
+ *       copyright notice, this list of conditions and the
+ *       following disclaimer.
+ *     * Redistributions in binary form must reproduce the
+ *       above copyright notice, this list of conditions and
+ *       the following disclaimer in the documentation and/or
+ *       other materials provided with the distribution.
+ *     * The names of contributors to this software may not be
+ *       used to endorse or promote products derived from this
+ *       software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ * Author: Stef Walter <stefw@redhat.com>
+ */
+
+#include "config.h"
+#include "CuTest.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "compat.h"
+#include "debug.h"
+#include "lexer.h"
+#include "library.h"
+#include "pem.h"
+
+typedef struct {
+	int tok_type;
+	const char *name;
+	const char *value;
+} expected_tok;
+
+static void
+on_pem_get_type (const char *type,
+                 const unsigned char *contents,
+                 size_t length,
+                 void *user_data)
+{
+	char **result = (char **)user_data;
+	*result = strdup (type);
+}
+
+static void
+check_lex_msg (CuTest *tc,
+               const char *file,
+               int line,
+               const expected_tok *expected,
+               const char *input,
+               bool failure)
+{
+	unsigned int count;
+	p11_lexer lexer;
+	char *type;
+	bool failed;
+	int i;
+
+	p11_lexer_init (&lexer, "test", input, strlen (input));
+	for (i = 0; p11_lexer_next (&lexer, &failed); i++) {
+		CuAssertIntEquals_LineMsg (tc, file, line,
+		                           "lexer token type does not match",
+		                           expected[i].tok_type, lexer.tok_type);
+		switch (lexer.tok_type) {
+		case TOK_FIELD:
+			CuAssertStrEquals_LineMsg (tc, file, line,
+			                           "field name doesn't match",
+			                           expected[i].name, lexer.tok.field.name);
+			CuAssertStrEquals_LineMsg (tc, file, line,
+			                           "field value doesn't match",
+			                           expected[i].value, lexer.tok.field.value);
+			break;
+		case TOK_SECTION:
+			CuAssertStrEquals_LineMsg (tc, file, line,
+			                           "section name doesn't match",
+			                           expected[i].name, lexer.tok.field.name);
+			break;
+		case TOK_PEM:
+			type = NULL;
+			count = p11_pem_parse (lexer.tok.pem.begin, lexer.tok.pem.length,
+			                       on_pem_get_type, &type);
+			CuAssertIntEquals_LineMsg (tc, file, line,
+			                           "wrong number of PEM blocks",
+			                           1, count);
+			CuAssertStrEquals_LineMsg (tc, file, line,
+			                           "wrong type of PEM block",
+			                           expected[i].name, type);
+			free (type);
+			break;
+		case TOK_EOF:
+			CuFail_Line (tc, file, line, NULL, "eof should not be recieved");
+			break;
+		}
+	}
+
+	if (failure)
+		CuAssert_Line (tc, file, line, "lexing didn't fail", failed);
+	else
+		CuAssert_Line (tc, file, line, "lexing failed", !failed);
+	CuAssertIntEquals_LineMsg (tc, file, line,
+	                           "premature end of lexing",
+	                           TOK_EOF, expected[i].tok_type);
+
+	p11_lexer_done (&lexer);
+}
+
+#define check_lex_success(tc, expected, input) \
+	check_lex_msg (tc, __FILE__, __LINE__, expected, input, false)
+
+#define check_lex_failure(tc, expected, input) \
+	check_lex_msg (tc, __FILE__, __LINE__, expected, input, true)
+
+static void
+test_basic (CuTest *tc)
+{
+	const char *input = "[the header]\n"
+	                    "field: value\n"
+	                    "-----BEGIN BLOCK1-----\n"
+	                    "aYNNXqshlVxCdo8QfKeXh3GUzd/yn4LYIVgQrx4a\n"
+	                    "-----END BLOCK1-----\n";
+
+	const expected_tok expected[] = {
+		{ TOK_SECTION, "the header" },
+		{ TOK_FIELD, "field", "value" },
+		{ TOK_PEM, "BLOCK1", },
+		{ TOK_EOF }
+	};
+
+	check_lex_success (tc, expected, input);
+}
+
+static void
+test_corners (CuTest *tc)
+{
+	const char *input = "\r\n"                 /* blankline */
+	                    " [the header]\r\n"    /* bad line endings */
+	                    "  field: value  \r\n" /* whitespace */
+	                    "number:    2\n"       /* extra space*/
+	                    "number    :3\n"       /* extra space*/
+	                    "number  :  4\n"       /* extra space*/
+	                    "\n"
+	                    " # A comment \n"
+	                    "not-a-comment: # value\n"
+	                    "-----BEGIN BLOCK1-----\r\n"
+	                    "aYNNXqshlVxCdo8QfKeXh3GUzd/yn4LYIVgQrx4a\r\n"
+	                    "-----END BLOCK1-----"; /* no new line */
+
+	const expected_tok expected[] = {
+		{ TOK_SECTION, "the header" },
+		{ TOK_FIELD, "field", "value" },
+		{ TOK_FIELD, "number", "2" },
+		{ TOK_FIELD, "number", "3" },
+		{ TOK_FIELD, "number", "4" },
+		{ TOK_FIELD, "not-a-comment", "# value" },
+		{ TOK_PEM, "BLOCK1", },
+		{ TOK_EOF }
+	};
+
+	check_lex_success (tc, expected, input);
+}
+
+static void
+test_following (CuTest *tc)
+{
+	const char *input = "-----BEGIN BLOCK1-----\n"
+	                    "aYNNXqshlVxCdo8QfKeXh3GUzd/yn4LYIVgQrx4a\n"
+	                    "-----END BLOCK1-----\n"
+	                    "field: value";
+
+	const expected_tok expected[] = {
+		{ TOK_PEM, "BLOCK1", },
+		{ TOK_FIELD, "field", "value" },
+		{ TOK_EOF }
+	};
+
+	check_lex_success (tc, expected, input);
+}
+
+static void
+test_bad_pem (CuTest *tc)
+{
+	const char *input = "field: value\n"
+	                    "-----BEGIN BLOCK1-----\n"
+	                    "aYNNXqshlVxCdo8QfKeXh3GUzd/yn4LYIVgQrx4a\n";
+
+	const expected_tok expected[] = {
+		{ TOK_FIELD, "field", "value" },
+		{ TOK_EOF }
+	};
+
+	p11_message_quiet ();
+
+	check_lex_failure (tc, expected, input);
+
+	p11_message_loud ();
+}
+
+static void
+test_bad_section (CuTest *tc)
+{
+	const char *input = "field: value\n"
+	                    "[section\n"
+	                    "bad]\n";
+
+	const expected_tok expected[] = {
+		{ TOK_FIELD, "field", "value" },
+		{ TOK_EOF }
+	};
+
+	p11_message_quiet ();
+
+	check_lex_failure (tc, expected, input);
+
+	p11_message_loud ();
+}
+
+static void
+test_bad_value (CuTest *tc)
+{
+	const char *input = "field_value\n"
+	                    "[section\n"
+	                    "bad]\n";
+
+	const expected_tok expected[] = {
+		{ TOK_EOF }
+	};
+
+	p11_message_quiet ();
+
+	check_lex_failure (tc, expected, input);
+
+	p11_message_loud ();
+}
+
+int
+main (void)
+{
+	CuString *output = CuStringNew ();
+	CuSuite* suite = CuSuiteNew ();
+	int ret;
+
+	putenv ("P11_KIT_STRICT=1");
+	p11_debug_init ();
+	p11_library_init ();
+
+	SUITE_ADD_TEST (suite, test_basic);
+	SUITE_ADD_TEST (suite, test_corners);
+	SUITE_ADD_TEST (suite, test_following);
+	SUITE_ADD_TEST (suite, test_bad_pem);
+	SUITE_ADD_TEST (suite, test_bad_section);
+	SUITE_ADD_TEST (suite, test_bad_value);
+
+	CuSuiteRun (suite);
+	CuSuiteSummary (suite, output);
+	CuSuiteDetails (suite, output);
+	printf ("%s\n", output->buffer);
+	ret = suite->failCount;
+	CuSuiteDelete (suite);
+	CuStringDelete (output);
+
+	return ret;
+}