buffer.compiler source code

1 module buffer.compiler;
2 
3 import std.string;
4 import std.conv;
5 import std.array;
6 import std.typecons;
7 import std.algorithm.searching;
8 import std.uni;
9 import std.exception;
10 
11 template LoadBufferFile(const string fileName)
12 {
13     pragma(msg, "Compiling file: ", fileName, "...");
14     const char[] LoadBufferFile = compiler!(import(fileName));
15 }
16 
17 template LoadBufferScript(const string src)
18 {
19     pragma(msg, "Compiling script: ", extractScriptfragment(src), "...");
20     const char[] LoadBufferScript = compiler!src;
21 }
22 
23 private string compiler(const string source)()
24 {
25     Token[] tokens = lexer(source);
26     Sentence[] sentences = parser(tokens);
27 
28     Appender!string code;
29     code.put("import buffer;\r\n\r\n");
30 
31     foreach (sentence; sentences)
32     {
33         code.put("final static class " ~ sentence.name ~ " : Message\r\n");
34         code.put("{\r\n");
35 
36         foreach (field; sentence.fields)
37         {
38             code.put("\t" ~ field.get.type ~ " " ~ field.get.name ~ ";\r\n");
39         }
40 
41         code.put("\r\n");
42         code.put("\tubyte[] serialize(const string method = string.init)\r\n");
43         code.put("\t{\r\n");
44         code.put("\t\treturn super.serialize!(typeof(this))(this, method);\r\n");
45         code.put("\t}\r\n");
46         code.put("}\r\n");
47         code.put("\r\n");
48     }
49 
50     return code.data;
51 }
52 
53 /// lexer
54 
55 enum TokenType
56 {
57     Define            = 1,           // message
58     Keyword           = 2,           // type: int8...
59     Identifier        = 3,
60     SentenceEnd       = 110,         // ;
61     DelimiterOpen     = 120,         // {
62     DelimiterClose    = 121          // }
63 }
64 
65 private const string[] keywords = [
66     "int8", "uint8", "int16", "uint16", "int32", "uint32", "int64", "uint64",
67     "float32", "float64", "float128", "bool", "char", "string"
68 ];
69 
70 struct Token
71 {
72     TokenType type;
73     string name;
74 
75     this(const string name)
76     {
77         if (name == "message")
78         {
79             this.type = TokenType.Define;
80         }
81         else if (keywords.any!(x => x == name))
82         {
83             this.type = TokenType.Keyword;
84         }
85         else
86         {
87             this.type = TokenType.Identifier;
88         }
89 
90         this.name = name;
91     }
92 
93     this(const TokenType type, const string name)
94     {
95         this.type = type;
96         this.name = name;
97     }
98 }
99 
100 Token[] lexer(const string source)
101 {
102     /* State transition diagram:
103     0:  none      1: word      2: {      3: ;      4: }
104     -1: /        -2: //       -3: /*
105 
106     0   -> \s[ \f\n\r\t\v]      0
107         -> A..Za..z_            1
108         -> {                    2 -> add token -> 0
109         -> ;                    3 -> add token -> 0
110         -> }                    4 -> add token -> 0
111         -> /                    hang state, -1
112         -> other                Exception
113     1   -> \s[ \f\n\r\t\v]      1 -> add token -> 0
114         -> A..Za..z0..9_        1
115         -> {                    1 -> add token -> 2 -> add token -> 0
116         -> ;                    1 -> add token -> 3 -> add token -> 0
117         -> }                    1 -> add token -> 4 -> add token -> 0
118         -> /                    hang state, -1
119         -> other                Exception
120     2   ->                      0
121     3   ->                      0
122     4   ->                      0
123    -1   -> /                    -2
124         -> *                    -3
125         -> other                Exception
126    -2   -> \n                   restore state, hang = 0
127         -> other                skip
128    -3   -> /                    if last is * then restore state & hang = 0, else skip
129         -> other                skip
130     */
131 
132     Token[] tokens;
133     int state = 0;
134     int stateHang;
135     char last;
136     string token = string.init;
137 
138     const string src = (cast(string) source ~ "\r\n");
139     foreach (i, ch; src)
140     {
141         switch (state)
142         {
143             case 0:
144                 if (isWhite(ch))
145                 {
146                     continue;
147                 }
148                 else if (isIdentifierFirstChar(ch))
149                 {
150                     token = ch.to!string;
151                     state = 1;
152                 }
153                 else if (ch == '{')
154                 {
155                     tokens ~= Token(TokenType.DelimiterOpen, "{");
156                     state = 0;
157                 }
158                 else if (ch == ';')
159                 {
160                     tokens ~= Token(TokenType.SentenceEnd, ";");
161                     state = 0;
162                 }
163                 else if (ch == '}')
164                 {
165                     tokens ~= Token(TokenType.DelimiterClose, "}");
166                     state = 0;
167                 }
168                 else if (ch == '/')
169                 {
170                     stateHang = state;
171                     state = -1;
172                 }
173                 else
174                 {
175                     enforce(0, "Invalid character: " ~ ch.to!string);
176                 }
177                 break;
178             case 1:
179                 if (isWhite(ch))
180                 {
181                     tokens ~= Token(token);
182                     token = string.init;
183                     state = 0;
184                 }
185                 else if (isIdentifierChar(ch))
186                 {
187                     token ~= ch.to!string;
188                 }
189                 else if (ch == '{')
190                 {
191                     tokens ~= Token(token);
192                     tokens ~= Token(TokenType.DelimiterOpen, "{");
193                     token = string.init;
194                     state = 0;
195                 }
196                 else if (ch == ';')
197                 {
198                     tokens ~= Token(token);
199                     tokens ~= Token(TokenType.SentenceEnd, ";");
200                     token = string.init;
201                     state = 0;
202                 }
203                 else if (ch == '}')
204                 {
205                     tokens ~= Token(token);
206                     tokens ~= Token(TokenType.DelimiterClose, "}");
207                     token = string.init;
208                     state = 0;
209                 }
210                 else if (ch == '/')
211                 {
212                     stateHang = state;
213                     state = -1;
214                 }
215                 else
216                 {
217                     enforce(0, "Invalid character: " ~ ch.to!string);
218                 }
219                 break;
220             case -1:
221                 if (ch == '/')
222                 {
223                     state = -2;
224                 }
225                 else if (ch == '*')
226                 {
227                     state = -3;
228                 }
229                 else
230                 {
231                     enforce(0, "Invalid character: " ~ ch.to!string);
232                 }
233                 break;
234             case -2:
235                 if (ch == '\n')
236                 {
237                     state = stateHang;
238                     stateHang = 0;
239                 }
240                 break;
241             case -3:
242                 if ((ch == '/') && (last == '*'))
243                 {
244                     state = stateHang;
245                     stateHang = 0;
246                 }
247                 break;
248             default:
249                 break;
250         }
251 
252         last = ch;
253     }
254 
255     return tokens;
256 }
257 
258 private bool isIdentifierFirstChar(const char ch)
259 {
260     return isAlpha(ch) || ch == '_';
261 }
262 
263 private bool isIdentifierChar(const char ch)
264 {
265     return isAlphaNum(ch) || ch == '_';
266 }
267 
268 private string extractScriptfragment(const string source)
269 {
270     string ret = string.init;
271 
272     foreach (ch; source)
273     {
274         if (ret.length >= 50)
275             break;
276         if (!isWhite(ch))
277             ret ~= ch.to!string;
278         else if ((ret.length > 0) && (ret[$ - 1] != ' '))
279             ret ~= " ";
280     }
281 
282     return ret;
283 }
284 
285 /// parser
286 
287 struct Field
288 {
289     string type;
290     string name;
291 }
292 
293 struct Sentence
294 {
295     string name;
296     Nullable!Field[] fields;
297 }
298 
299 Sentence[] parser(const Token[] tokens)
300 {
301     Sentence[] sentences;
302     int pos;
303     while (pos < cast(int)tokens.length - 1)
304     {
305         if (tokens[pos].type != TokenType.Define)
306         {
307             enforce(0, "Syntax error at " ~ tokens[pos].name);
308         }
309 
310         sentences ~= parser_define(tokens, pos);
311     }
312 
313     return sentences;
314 }
315 
316 private Sentence parser_define(const Token[] tokens, ref int pos)
317 {
318     if ((cast(int)tokens.length - pos < 4) || (tokens[pos].type != TokenType.Define) || (tokens[pos + 1].type != TokenType.Identifier) || (tokens[pos + 2].type != TokenType.DelimiterOpen))
319     {
320         enforce(0, "Syntax error at " ~ tokens[pos].name);
321     }
322 
323     Sentence sentence;
324     sentence.name = tokens[pos + 1].name;
325     pos += 3;
326 
327     while (pos < tokens.length)
328     {
329         Nullable!Field field = parser_field(tokens, pos);
330 
331         if (field.isNull)
332             return sentence;
333 
334         sentence.fields ~= field;
335     }
336 
337     return sentence;
338 }
339 
340 private Nullable!Field parser_field(const Token[] tokens, ref int pos)
341 {
342     if ((cast(int)tokens.length - pos >= 1) && (tokens[pos].type == TokenType.DelimiterClose))
343     {
344         pos++;
345         return Nullable!Field();
346     }
347 
348     if ((cast(int)tokens.length - pos < 3) || (tokens[pos].type != TokenType.Keyword)
349             || (tokens[pos + 1].type != TokenType.Identifier)
350             || (tokens[pos + 2].type != TokenType.SentenceEnd))
351     {
352         enforce(0, "Syntax error at " ~ tokens[pos].name);
353     }
354 
355     Field field;
356     field.type = tokens[pos].name;
357     field.name = tokens[pos + 1].name;
358     pos += 3;
359 
360     return Nullable!Field(field);
361 }
362 
363 /*
364 import buffer;
365 
366 final static class Sample : Message
367 {
368     string name;
369     int32 age;
370     int16 sex;
371 
372     ubyte[] serialize(const string method = string.init)
373     {
374         return super.serialize!(typeof(this))(this, method);
375     }
376 }
377 */