nk: prototype ttl lexer.

7 files changed, 115 insertions, 181 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 22803e6..1f33348 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -127,15 +127,16 @@ if(NOT WIN32)
 endif()
 install(TARGETS sherlock DESTINATION ${PLUGIN_DEST})
 
-#find_package(FLEX)
-#flex_target(encoder encoder.l ${PROJECT_BINARY_DIR}/encoder.c
-#	COMPILE_FLAGS	"--header-file=${PROJECT_BINARY_DIR}/encoder.h --prefix=enc")
+find_package(FLEX)
+flex_target(encoder encoder.l ${PROJECT_BINARY_DIR}/encoder.c
+	COMPILE_FLAGS	"--prefix=enc")
 
 add_library(sherlock_nk MODULE
 	sherlock_nk.c
 	midi_inspector_nk.c
 	atom_inspector_nk.c
 	osc_inspector_nk.c
+	encoder
 	${TAR_UI})
 target_link_libraries(sherlock_nk
 	${LIBS_UI})
diff --git a/VERSION b/VERSION
index ce328cb..4ec9ab5 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.11.2821
+0.11.2823
diff --git a/atom_inspector_nk.c b/atom_inspector_nk.c
index 07b1a9e..1551df5 100644
--- a/atom_inspector_nk.c
+++ b/atom_inspector_nk.c
@@ -21,6 +21,8 @@
 #include <sherlock.h>
 #include <sherlock_nk.h>
 
+#include <encoder.h>
+
 #define NS_RDF (const uint8_t*)"http://www.w3.org/1999/02/22-rdf-syntax-ns#"
 #define NS_RDFS (const uint8_t*)"http://www.w3.org/2000/01/rdf-schema#"
 #define NS_XSD (const uint8_t*)"http://www.w3.org/2001/XMLSchema#"
@@ -88,6 +90,24 @@ _sratom_to_turtle(Sratom*         sratom,
 	return (char*)serd_chunk_sink_finish(&str);
 }
 
+static void
+_set_string(struct nk_str *str, uint32_t size, const char *body)
+{
+	nk_str_clear(str);
+
+	// replace tab with 2 spaces
+	const char *end = body + size - 1;
+	const char *from = body;
+	for(const char *to = strchr(from, '\t');
+		to && (to < end);
+		from = to + 1, to = strchr(from, '\t'))
+	{
+		nk_str_append_text_utf8(str, from, to-from);
+		nk_str_append_text_utf8(str, "  ", 2);
+	}
+	nk_str_append_text_utf8(str, from, end-from);
+}
+
 void
 _atom_inspector_expose(struct nk_context *ctx, struct nk_rect wbounds, void *data)
 {
@@ -237,20 +257,12 @@ _atom_inspector_expose(struct nk_context *ctx, struct nk_rect wbounds, void *dat
 					atom->type, atom->size, LV2_ATOM_BODY_CONST(atom));
 				if(ttl)
 				{
-					struct nk_str *str = &handle->str;
+					struct nk_str *str = &handle->editor.string;
 					const size_t len = strlen(ttl);
 
-					nk_str_clear(str);
+					_set_string(str, len, ttl);
 
-					char *from, *to;
-					for(from=ttl, to=strchr(from, '\t');
-						to;
-						from=to+1, to=strchr(from, '\t'))
-					{
-						nk_str_append_text_utf8(str, from, to-from);
-						nk_str_append_text_utf8(str, "  ", 2);
-					}
-					nk_str_append_text_utf8(str, from, strlen(from));
+					handle->editor.lexer.needs_refresh = 1;
 
 					free(ttl);
 				}
@@ -259,15 +271,15 @@ _atom_inspector_expose(struct nk_context *ctx, struct nk_rect wbounds, void *dat
 			}
 
 			const nk_flags flags = NK_EDIT_EDITOR;
-			char *str = nk_str_get(&handle->str);
-			int len = nk_str_len(&handle->str);
+			char *str = nk_str_get(&handle->editor.string);
+			int len = nk_str_len(&handle->editor.string);
 
 			if(len > 0) //FIXME
 			{
 				const float content_h = nk_window_get_height(ctx) - 2*window_padding.y - 2*group_padding.y;
 				nk_layout_row_dynamic(ctx, content_h, 1);
 				nk_edit_focus(ctx, flags);
-				const nk_flags mode = nk_edit_string(ctx, flags, str, &len, len, nk_filter_default);
+				const nk_flags mode = nk_edit_buffer(ctx, flags, &handle->editor, nk_filter_default);
 				(void)mode;
 			}
 
diff --git a/encoder.h b/encoder.h
new file mode 100644
index 0000000..9d376c6
--- /dev/null
+++ b/encoder.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2015-2016 Hanspeter Portner (dev@open-music-kontrollers.ch)
+ *
+ * This is free software: you can redistribute it and/or modify
+ * it under the terms of the Artistic License 2.0 as published by
+ * The Perl Foundation.
+ *
+ * This source is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Artistic License 2.0 for more details.
+ *
+ * You should have received a copy of the Artistic License 2.0
+ * along the source as a COPYING file. If not, obtain it from
+ * http://www.perlfoundation.org/artistic_license_2_0.
+ */
+
+#ifndef ENCODER_H
+#define ENCODER_H
+
+#include "nk_pugl/nk_pugl.h"
+
+struct nk_token *
+ttl_lex(void *data, const char *utf8, int len);
+
+#endif
diff --git a/encoder.l b/encoder.l
index d076f67..50ac8b3 100644
--- a/encoder.l
+++ b/encoder.l
@@ -19,76 +19,7 @@
 #include <stdio.h>
 #include <string.h>
 
-#include <common.h>
-
-typedef enum _markup_type_t markup_type_t;
-typedef struct _markup_item_t markup_item_t;
-
-enum _markup_type_t {
-	MARKUP_CODE,
-	MARKUP_PREFIX,
-	MARKUP_SUBJECT,
-	MARKUP_PREDICATE,
-	MARKUP_NUMBER,
-	MARKUP_STRING,
-	MARKUP_URI
-};
-
-struct _markup_item_t {
-	const char *begin;
-	const char *end;
-};
-
-static const markup_item_t markup_items [] = {
-	[MARKUP_CODE] =				{"font=Mono style=Plain color=#ffffff", "font"},
-	[MARKUP_PREFIX] =			{"color=#cc00cc", "color"},
-	[MARKUP_SUBJECT] =		{"color=#00cccc", "color"},
-	[MARKUP_PREDICATE] =	{"color=#00cc00", "color"},
-	[MARKUP_NUMBER] =			{"color=#0000cc", "color"},
-	[MARKUP_STRING] =			{"color=#cc0000", "color"},
-	[MARKUP_URI] =				{"color=#cccc00", "color"}
-};
-
-static void
-_add_plain(const char *content)
-{
-	encoder->append(content, encoder->data);
-}
-
-static void 
-_add_singleton(const char *key)
-{
-	char buf [64];
-	sprintf(buf, "<%s/>", key);
-	encoder->append(buf, encoder->data);
-}
-
-static void 
-_add_markup_begin(markup_type_t type)
-{
-	char buf [64];
-	sprintf(buf, "<%s>", markup_items[type].begin);
-	encoder->append(buf, encoder->data);
-}
-
-static void 
-_add_markup_end(markup_type_t type)
-{
-	char buf [64];
-	sprintf(buf, "</%s>", markup_items[type].end);
-	encoder->append(buf, encoder->data);
-}
-
-static void 
-_add_markup(markup_type_t type, const char *content)
-{
-	char buf [64];
-	sprintf(buf, "<%s>", markup_items[type].begin);
-	encoder->append(buf, encoder->data);
-	encoder->append(content, encoder->data);
-	sprintf(buf, "</%s>", markup_items[type].end);
-	encoder->append(buf, encoder->data);
-}
+#include <encoder.h>
 
 enum {
 	TK_NONE,
@@ -105,14 +36,7 @@ enum {
 	TK_LONG_STRING_IN,
 	TK_LONG_STRING_OUT,
 	TK_WHITESPACE,
-	TK_RAW,
-	TK_TAB,
-	TK_NEWLINE,
-	TK_LT,
-	TK_GT,
-	TK_AMP,
-	TK_NAME,
-	TK_BADCHAR
+	TK_RAW
 };
 
 %}
@@ -133,8 +57,7 @@ eol            [\n\r]
 %%
 
 {w}            return TK_WHITESPACE;
-"\t"           return TK_TAB;
-{eol}          return TK_NEWLINE;
+"\t"           return TK_WHITESPACE;
 "<"            BEGIN(XURI); return TK_URI_IN;
 \"\"\"         BEGIN(XLONG_STRING); return TK_LONG_STRING_IN;
 \"             BEGIN(XSTRING); return TK_STRING_IN;
@@ -157,11 +80,6 @@ eol            [\n\r]
    \\\"	       return TK_RAW;
    \"\"\"      BEGIN(0); return TK_LONG_STRING_OUT;
    {w}         return TK_WHITESPACE;
-   "\t"        return TK_TAB;
-   {eol}       return TK_NEWLINE;
-   "<"         return TK_LT;
-   ">"         return TK_GT;
-   "&"         return TK_AMP;
    .           return TK_RAW;
 }
 
@@ -171,18 +89,13 @@ eol            [\n\r]
    \"          BEGIN(0); return TK_STRING_OUT;
    {eol}	     BEGIN(0); return TK_STRING_ERR;
    {w}         return TK_WHITESPACE;
-   "\t"        return TK_TAB;
-   {eol}       return TK_NEWLINE;
-   "<"         return TK_LT;
-   ">"         return TK_GT;
-   "&"         return TK_AMP;
    .           return TK_RAW;
 }
 
 %%
 
-void
-ttl_to_markup(const char *utf8, FILE *f)
+struct nk_token *
+ttl_lex(void *data, const char *utf8, int len)
 {
 	yyscan_t scanner;
 	YY_BUFFER_STATE buf;
@@ -190,109 +103,83 @@ ttl_to_markup(const char *utf8, FILE *f)
 	enclex_init(&scanner);
 	if(utf8)
 	{
-		buf = enc_scan_string(utf8, scanner);
-	}
-	else if(f)
-	{
-		encset_in(f, scanner);
-		buf = enc_create_buffer(NULL, YY_BUF_SIZE, scanner);
+		buf = enc_scan_bytes(utf8, len, scanner);
 	}
 	else
 	{
 		enclex_destroy(scanner);
-		return;
+		return NULL;
 	}
+
+	struct nk_token *tokens = NULL;
+	int n_tokens = 0;
 				
-	encoder->begin(encoder->data);
-	_add_markup_begin(MARKUP_CODE);
+	const char *base = encget_text(scanner);
+	int offset0  = 0;
+	struct nk_color col0 = {0xff, 0xff, 0xff, 0xff};
 
 	for(int tok=enclex(scanner); tok; tok=enclex(scanner))
 	{
 		const char *txt = encget_text(scanner);
+		const int offset1 = txt - base;
+		struct nk_color col1 = col0;
+
 		switch(tok)
 		{
 			case TK_PREFIX:
-				_add_markup(MARKUP_PREFIX, txt);
+				col1 = (struct nk_color){0xff, 0x00, 0x00, 0xff};
+				break;
+			case TK_SUBJECT:
+				col1 = (struct nk_color){0x00, 0xff, 0x00, 0xff};
+				break;
+			case TK_PREDICATE:
+				col1 = (struct nk_color){0x00, 0x00, 0xff, 0xff};
 				break;
-
 			case TK_NUMBER:
-				_add_markup(MARKUP_NUMBER, txt);
+				col1 = (struct nk_color){0xff, 0xff, 0x00, 0xff};
 				break;
-
 			case TK_URI_IN:
-				_add_markup_begin(MARKUP_URI);
-				_add_plain("&lt;");
-				break;
 			case TK_URI_OUT:
-				_add_plain("&gt;");
-				_add_markup_end(MARKUP_URI);
-				break;
 			case TK_URI_ERR:
-				_add_markup_end(MARKUP_URI);
-				_add_singleton("br");
+				col1 = (struct nk_color){0xff, 0x00, 0xff, 0xff};
 				break;
 
 			case TK_STRING_IN:
-				_add_markup_begin(MARKUP_STRING);
-				_add_plain("\"");
-				break;
 			case TK_STRING_OUT:
-				_add_plain("\"");
-				_add_markup_end(MARKUP_STRING);
-				break;
 			case TK_STRING_ERR:
-				_add_markup_end(MARKUP_STRING);
-				_add_singleton("br");
-				break;
-
 			case TK_LONG_STRING_IN:
-				_add_markup_begin(MARKUP_STRING);
-				_add_plain("\"\"\"");
-				break;
 			case TK_LONG_STRING_OUT:
-				_add_plain("\"\"\"");
-				_add_markup_end(MARKUP_STRING);
+				col1 = (struct nk_color){0x00, 0xff, 0xff, 0xff};
 				break;
 
-			case TK_NEWLINE:
-				_add_singleton("br");
-				break;
-			case TK_LT:
-				_add_plain("&lt;");
-				break;
-			case TK_GT:
-				_add_plain("&gt;");
-				break;
-			case TK_AMP:
-				_add_plain("&amp;");
-				break;
-			
-			case TK_BADCHAR:
-				break;
-			
-			case TK_TAB:
-				_add_plain("  ");
-				break;
-
-			case TK_SUBJECT:
-			  _add_markup(MARKUP_SUBJECT, txt);
-				break;
-			case TK_PREDICATE:
-			  _add_markup(MARKUP_PREDICATE, txt);
+			case TK_NONE:
+			case TK_WHITESPACE:
+				col1 = (struct nk_color){0xff, 0xff, 0xff, 0xff};
 				break;
 
-			case TK_NAME:
-			case TK_WHITESPACE:
 			case TK_RAW:
 			default:
-				_add_plain(txt);
+				// skip over
 				break;
 		}
+
+		if(offset1)
+		{
+			tokens = realloc(tokens, (n_tokens + 1) * sizeof(struct nk_token));
+			tokens[n_tokens].offset = offset1;
+			tokens[n_tokens++].color = col0;
+		}
+
+		offset0 = offset1;
+		col0 = col1;
 	}
-				
-	_add_markup_end(MARKUP_CODE);
-	encoder->end(encoder->data);
+
+	tokens = realloc(tokens, (n_tokens + 1) * sizeof(struct nk_token));
+	tokens[n_tokens].offset = len;
+	tokens[n_tokens++].color = (struct nk_color){0xff, 0xff, 0xff, 0xff};
 
 	enc_delete_buffer(buf, scanner);
 	enclex_destroy(scanner);
+
+	return tokens;
 }
diff --git a/sherlock_nk.c b/sherlock_nk.c
index d82c357..8684f7c 100644
--- a/sherlock_nk.c
+++ b/sherlock_nk.c
@@ -24,6 +24,8 @@
 #include <sherlock.h>
 #include <osc.lv2/util.h>
 
+#include <encoder.h>
+
 #define NK_PUGL_IMPLEMENTATION
 #include <sherlock_nk.h>
 
@@ -222,7 +224,8 @@ void
 _clear(plughandle_t *handle)
 {
 	_clear_items(handle);
-	nk_str_clear(&handle->str);
+	struct nk_str *str = &handle->editor.string;
+	nk_str_clear(str);
 	handle->selected = NULL;
 	handle->counter = 1;
 }
@@ -341,7 +344,9 @@ instantiate(const LV2UI_Descriptor *descriptor, const char *plugin_uri,
 	_clear(handle);
 	_discover(handle);
 
-	nk_str_init_default(&handle->str);
+	nk_textedit_init_default(&handle->editor);
+	handle->editor.lexer.lex = ttl_lex;
+	handle->editor.lexer.data = NULL;
 
 	handle->sratom = sratom_new(handle->map);
 	sratom_set_pretty_numbers(handle->sratom, false);
@@ -358,7 +363,10 @@ cleanup(LV2UI_Handle instance)
 	sratom_free(handle->sratom);
 
 	_clear_items(handle);
-	nk_str_free(&handle->str);
+	nk_textedit_free(&handle->editor);
+
+	if(handle->editor.lexer.tokens)
+		free(handle->editor.lexer.tokens);
 
 	nk_pugl_hide(&handle->win);
 	nk_pugl_shutdown(&handle->win);
diff --git a/sherlock_nk.h b/sherlock_nk.h
index 0ec23cf..a81bbad 100644
--- a/sherlock_nk.h
+++ b/sherlock_nk.h
@@ -93,7 +93,7 @@ struct _plughandle_t {
 
 	bool ttl_dirty;
 	const LV2_Atom *selected;
-	struct nk_str str;
+	struct nk_text_edit editor;
 
 	Sratom *sratom;
 	const char *base_uri;