Coverage for C:\Repos\leo-editor\leo\core\leoImport.py: 13%
1854 statements
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20031218072017.3206: * @file leoImport.py
4#@@first
5#@+<< imports >>
6#@+node:ekr.20091224155043.6539: ** << imports >> (leoImport)
7import csv
8import io
9import json
10import os
11import re
12import textwrap
13import time
14from typing import Any, List
15import urllib
16#
17# Third-party imports.
18try:
19 import docutils
20 import docutils.core
21 assert docutils
22 assert docutils.core
23except ImportError:
24 # print('leoImport.py: can not import docutils')
25 docutils = None # type:ignore
26try:
27 import lxml
28except ImportError:
29 lxml = None
30#
31# Leo imports...
32from leo.core import leoGlobals as g
33from leo.core import leoNodes
34#
35# Abbreviation.
36StringIO = io.StringIO
37#@-<< imports >>
38#@+others
39#@+node:ekr.20160503145550.1: ** class FreeMindImporter
40class FreeMindImporter:
41 """Importer class for FreeMind (.mmap) files."""
43 def __init__(self, c):
44 """ctor for FreeMind Importer class."""
45 self.c = c
46 self.count = 0
47 self.d = {}
48 #@+others
49 #@+node:ekr.20170222084048.1: *3* freemind.add_children
50 def add_children(self, parent, element):
51 """
52 parent is the parent position, element is the parent element.
53 Recursively add all the child elements as descendants of parent_p.
54 """
55 p = parent.insertAsLastChild()
56 attrib_text = element.attrib.get('text', '').strip()
57 tag = element.tag if isinstance(element.tag, str) else ''
58 text = element.text or ''
59 if not tag:
60 text = text.strip()
61 p.h = attrib_text or tag or 'Comment'
62 p.b = text if text.strip() else ''
63 for child in element:
64 self.add_children(p, child)
65 #@+node:ekr.20160503125844.1: *3* freemind.create_outline
66 def create_outline(self, path):
67 """Create a tree of nodes from a FreeMind file."""
68 c = self.c
69 junk, fileName = g.os_path_split(path)
70 undoData = c.undoer.beforeInsertNode(c.p)
71 try:
72 self.import_file(path)
73 c.undoer.afterInsertNode(c.p, 'Import', undoData)
74 except Exception:
75 g.es_print('Exception importing FreeMind file', g.shortFileName(path))
76 g.es_exception()
77 return c.p
78 #@+node:ekr.20160503191518.4: *3* freemind.import_file
79 def import_file(self, path):
80 """The main line of the FreeMindImporter class."""
81 c = self.c
82 sfn = g.shortFileName(path)
83 if g.os_path_exists(path):
84 htmltree = lxml.html.parse(path)
85 root = htmltree.getroot()
86 body = root.findall('body')[0]
87 if body is None:
88 g.error(f"no body in: {sfn}")
89 else:
90 root_p = c.lastTopLevel().insertAfter()
91 root_p.h = g.shortFileName(path)
92 for child in body:
93 if child != body:
94 self.add_children(root_p, child)
95 c.selectPosition(root_p)
96 c.redraw()
97 else:
98 g.error(f"file not found: {sfn}")
99 #@+node:ekr.20160503145113.1: *3* freemind.import_files
100 def import_files(self, files):
101 """Import a list of FreeMind (.mmap) files."""
102 c = self.c
103 if files:
104 self.tab_width = c.getTabWidth(c.p)
105 for fileName in files:
106 g.setGlobalOpenDir(fileName)
107 p = self.create_outline(fileName)
108 p.contract()
109 p.setDirty()
110 c.setChanged()
111 c.redraw(p)
112 #@+node:ekr.20160504043823.1: *3* freemind.prompt_for_files
113 def prompt_for_files(self):
114 """Prompt for a list of FreeMind (.mm.html) files and import them."""
115 if not lxml:
116 g.trace("FreeMind importer requires lxml")
117 return
118 c = self.c
119 types = [
120 ("FreeMind files", "*.mm.html"),
121 ("All files", "*"),
122 ]
123 names = g.app.gui.runOpenFileDialog(c,
124 title="Import FreeMind File",
125 filetypes=types,
126 defaultextension=".html",
127 multiple=True)
128 c.bringToFront()
129 if names:
130 g.chdir(names[0])
131 self.import_files(names)
132 #@-others
133#@+node:ekr.20160504144241.1: ** class JSON_Import_Helper
134class JSON_Import_Helper:
135 """
136 A class that helps client scripts import .json files.
138 Client scripts supply data describing how to create Leo outlines from
139 the .json data.
140 """
142 def __init__(self, c):
143 """ctor for the JSON_Import_Helper class."""
144 self.c = c
145 self.vnodes_dict = {}
146 #@+others
147 #@+node:ekr.20160504144353.1: *3* json.create_nodes (generalize)
148 def create_nodes(self, parent, parent_d):
149 """Create the tree of nodes rooted in parent."""
150 d = self.gnx_dict
151 for child_gnx in parent_d.get('children'):
152 d2 = d.get(child_gnx)
153 if child_gnx in self.vnodes_dict:
154 # It's a clone.
155 v = self.vnodes_dict.get(child_gnx)
156 n = parent.numberOfChildren()
157 child = leoNodes.Position(v)
158 child._linkAsNthChild(parent, n)
159 # Don't create children again.
160 else:
161 child = parent.insertAsLastChild()
162 child.h = d2.get('h') or '<**no h**>'
163 child.b = d2.get('b') or ''
164 if d2.get('gnx'):
165 child.v.fileIndex = gnx = d2.get('gnx') # 2021/06/23: found by mypy.
166 self.vnodes_dict[gnx] = child.v
167 if d2.get('ua'):
168 child.u = d2.get('ua')
169 self.create_nodes(child, d2)
170 #@+node:ekr.20160504144241.2: *3* json.create_outline (generalize)
171 def create_outline(self, path):
172 c = self.c
173 junk, fileName = g.os_path_split(path)
174 undoData = c.undoer.beforeInsertNode(c.p)
175 # Create the top-level headline.
176 p = c.lastTopLevel().insertAfter()
177 fn = g.shortFileName(path).strip()
178 if fn.endswith('.json'):
179 fn = fn[:-5]
180 p.h = fn
181 self.scan(path, p)
182 c.undoer.afterInsertNode(p, 'Import', undoData)
183 return p
184 #@+node:ekr.20160504144314.1: *3* json.scan (generalize)
185 def scan(self, s, parent):
186 """Create an outline from a MindMap (.csv) file."""
187 c, d, self.gnx_dict = self.c, json.loads(s), {}
188 for d2 in d.get('nodes', []):
189 gnx = d2.get('gnx')
190 self.gnx_dict[gnx] = d2
191 top_d = d.get('top')
192 if top_d:
193 # Don't set parent.h or parent.gnx or parent.v.u.
194 parent.b = top_d.get('b') or ''
195 self.create_nodes(parent, top_d)
196 c.redraw()
197 return bool(top_d)
198 #@-others
199#@+node:ekr.20071127175948: ** class LeoImportCommands
200class LeoImportCommands:
201 """
202 A class implementing all of Leo's import/export code. This class
203 uses **importers** in the leo/plugins/importers folder.
205 For more information, see leo/plugins/importers/howto.txt.
206 """
207 #@+others
208 #@+node:ekr.20031218072017.3207: *3* ic.__init__& ic.reload_settings
209 def __init__(self, c):
210 """ctor for LeoImportCommands class."""
211 self.c = c
212 self.encoding = 'utf-8'
213 self.errors = 0
214 self.fileName = None # The original file name, say x.cpp
215 self.fileType = None # ".py", ".c", etc.
216 self.methodName = None # x, as in < < x methods > > =
217 self.output_newline = g.getOutputNewline(c=c) # Value of @bool output_newline
218 self.tab_width = c.tab_width
219 self.treeType = "@file" # None or "@file"
220 self.verbose = True # Leo 6.6
221 self.webType = "@noweb" # "cweb" or "noweb"
222 self.web_st = [] # noweb symbol table.
223 self.reload_settings()
225 def reload_settings(self):
226 pass
228 reloadSettings = reload_settings
229 #@+node:ekr.20031218072017.3289: *3* ic.Export
230 #@+node:ekr.20031218072017.3290: *4* ic.convertCodePartToWeb & helpers
231 def convertCodePartToWeb(self, s, i, p, result):
232 """
233 # Headlines not containing a section reference are ignored in noweb
234 and generate index index in cweb.
235 """
236 ic = self
237 nl = ic.output_newline
238 head_ref = ic.getHeadRef(p)
239 file_name = ic.getFileName(p)
240 if g.match_word(s, i, "@root"):
241 i = g.skip_line(s, i)
242 ic.appendRefToFileName(file_name, result)
243 elif g.match_word(s, i, "@c") or g.match_word(s, i, "@code"):
244 i = g.skip_line(s, i)
245 ic.appendHeadRef(p, file_name, head_ref, result)
246 elif g.match_word(p.h, 0, "@file"):
247 # Only do this if nothing else matches.
248 ic.appendRefToFileName(file_name, result)
249 i = g.skip_line(s, i) # 4/28/02
250 else:
251 ic.appendHeadRef(p, file_name, head_ref, result)
252 i, result = ic.copyPart(s, i, result)
253 return i, result.strip() + nl
254 #@+at %defs a b c
255 #@+node:ekr.20140630085837.16720: *5* ic.appendHeadRef
256 def appendHeadRef(self, p, file_name, head_ref, result):
257 ic = self
258 nl = ic.output_newline
259 if ic.webType == "cweb":
260 if head_ref:
261 escaped_head_ref = head_ref.replace("@", "@@")
262 result += "@<" + escaped_head_ref + "@>=" + nl
263 else:
264 # Convert the headline to an index entry.
265 result += "@^" + p.h.strip() + "@>" + nl
266 result += "@c" + nl # @c denotes a new section.
267 else:
268 if head_ref:
269 pass
270 elif p == ic.c.p:
271 head_ref = file_name or "*"
272 else:
273 head_ref = "@others"
274 # 2019/09/12
275 result += (g.angleBrackets(head_ref) + "=" + nl)
276 #@+node:ekr.20140630085837.16719: *5* ic.appendRefToFileName
277 def appendRefToFileName(self, file_name, result):
278 ic = self
279 nl = ic.output_newline
280 if ic.webType == "cweb":
281 if not file_name:
282 result += "@<root@>=" + nl
283 else:
284 result += "@(" + file_name + "@>" + nl # @(...@> denotes a file.
285 else:
286 if not file_name:
287 file_name = "*"
288 # 2019/09/12.
289 lt = "<<"
290 rt = ">>"
291 result += (lt + file_name + rt + "=" + nl)
292 #@+node:ekr.20140630085837.16721: *5* ic.getHeadRef
293 def getHeadRef(self, p):
294 """
295 Look for either noweb or cweb brackets.
296 Return everything between those brackets.
297 """
298 h = p.h.strip()
299 if g.match(h, 0, "<<"):
300 i = h.find(">>", 2)
301 elif g.match(h, 0, "<@"):
302 i = h.find("@>", 2)
303 else:
304 return h
305 return h[2:i].strip()
306 #@+node:ekr.20031218072017.3292: *5* ic.getFileName
307 def getFileName(self, p):
308 """Return the file name from an @file or @root node."""
309 h = p.h.strip()
310 if g.match(h, 0, "@file") or g.match(h, 0, "@root"):
311 line = h[5:].strip()
312 # set j & k so line[j:k] is the file name.
313 if g.match(line, 0, "<"):
314 j, k = 1, line.find(">", 1)
315 elif g.match(line, 0, '"'):
316 j, k = 1, line.find('"', 1)
317 else:
318 j, k = 0, line.find(" ", 0)
319 if k == -1:
320 k = len(line)
321 file_name = line[j:k].strip()
322 else:
323 file_name = ''
324 return file_name
325 #@+node:ekr.20031218072017.3296: *4* ic.convertDocPartToWeb (handle @ %def)
326 def convertDocPartToWeb(self, s, i, result):
327 nl = self.output_newline
328 if g.match_word(s, i, "@doc"):
329 i = g.skip_line(s, i)
330 elif g.match(s, i, "@ ") or g.match(s, i, "@\t") or g.match(s, i, "@*"):
331 i += 2
332 elif g.match(s, i, "@\n"):
333 i += 1
334 i = g.skip_ws_and_nl(s, i)
335 i, result2 = self.copyPart(s, i, "")
336 if result2:
337 # Break lines after periods.
338 result2 = result2.replace(". ", "." + nl)
339 result2 = result2.replace(". ", "." + nl)
340 result += nl + "@" + nl + result2.strip() + nl + nl
341 else:
342 # All nodes should start with '@', even if the doc part is empty.
343 result += nl + "@ " if self.webType == "cweb" else nl + "@" + nl
344 return i, result
345 #@+node:ekr.20031218072017.3297: *4* ic.convertVnodeToWeb
346 def convertVnodeToWeb(self, v):
347 """
348 This code converts a VNode to noweb text as follows:
350 Convert @doc to @
351 Convert @root or @code to < < name > >=, assuming the headline contains < < name > >
352 Ignore other directives
353 Format doc parts so they fit in pagewidth columns.
354 Output code parts as is.
355 """
356 c = self.c
357 if not v or not c:
358 return ""
359 startInCode = not c.config.at_root_bodies_start_in_doc_mode
360 nl = self.output_newline
361 docstart = nl + "@ " if self.webType == "cweb" else nl + "@" + nl
362 s = v.b
363 lb = "@<" if self.webType == "cweb" else "<<"
364 i, result, docSeen = 0, "", False
365 while i < len(s):
366 progress = i
367 i = g.skip_ws_and_nl(s, i)
368 if self.isDocStart(s, i) or g.match_word(s, i, "@doc"):
369 i, result = self.convertDocPartToWeb(s, i, result)
370 docSeen = True
371 elif (
372 g.match_word(s, i, "@code") or
373 g.match_word(s, i, "@root") or
374 g.match_word(s, i, "@c") or
375 g.match(s, i, lb)
376 ):
377 if not docSeen:
378 docSeen = True
379 result += docstart
380 i, result = self.convertCodePartToWeb(s, i, v, result)
381 elif self.treeType == "@file" or startInCode:
382 if not docSeen:
383 docSeen = True
384 result += docstart
385 i, result = self.convertCodePartToWeb(s, i, v, result)
386 else:
387 i, result = self.convertDocPartToWeb(s, i, result)
388 docSeen = True
389 assert progress < i
390 result = result.strip()
391 if result:
392 result += nl
393 return result
394 #@+node:ekr.20031218072017.3299: *4* ic.copyPart
395 # Copies characters to result until the end of the present section is seen.
397 def copyPart(self, s, i, result):
399 lb = "@<" if self.webType == "cweb" else "<<"
400 rb = "@>" if self.webType == "cweb" else ">>"
401 theType = self.webType
402 while i < len(s):
403 progress = j = i # We should be at the start of a line here.
404 i = g.skip_nl(s, i)
405 i = g.skip_ws(s, i)
406 if self.isDocStart(s, i):
407 return i, result
408 if (g.match_word(s, i, "@doc") or
409 g.match_word(s, i, "@c") or
410 g.match_word(s, i, "@root") or
411 g.match_word(s, i, "@code") # 2/25/03
412 ):
413 return i, result
414 # 2019/09/12
415 lt = "<<"
416 rt = ">>="
417 if g.match(s, i, lt) and g.find_on_line(s, i, rt) > -1:
418 return i, result
419 # Copy the entire line, escaping '@' and
420 # Converting @others to < < @ others > >
421 i = g.skip_line(s, j)
422 line = s[j:i]
423 if theType == "cweb":
424 line = line.replace("@", "@@")
425 else:
426 j = g.skip_ws(line, 0)
427 if g.match(line, j, "@others"):
428 line = line.replace("@others", lb + "@others" + rb)
429 elif g.match(line, 0, "@"):
430 # Special case: do not escape @ %defs.
431 k = g.skip_ws(line, 1)
432 if not g.match(line, k, "%defs"):
433 line = "@" + line
434 result += line
435 assert progress < i
436 return i, result.rstrip()
437 #@+node:ekr.20031218072017.1462: *4* ic.exportHeadlines
438 def exportHeadlines(self, fileName):
439 p = self.c.p
440 nl = self.output_newline
441 if not p:
442 return
443 self.setEncoding()
444 firstLevel = p.level()
445 try:
446 with open(fileName, 'w') as theFile:
447 for p in p.self_and_subtree(copy=False):
448 head = p.moreHead(firstLevel, useVerticalBar=True)
449 theFile.write(head + nl)
450 except IOError:
451 g.warning("can not open", fileName)
452 #@+node:ekr.20031218072017.1147: *4* ic.flattenOutline
453 def flattenOutline(self, fileName):
454 """
455 A helper for the flatten-outline command.
457 Export the selected outline to an external file.
458 The outline is represented in MORE format.
459 """
460 c = self.c
461 nl = self.output_newline
462 p = c.p
463 if not p:
464 return
465 self.setEncoding()
466 firstLevel = p.level()
467 try:
468 theFile = open(fileName, 'wb') # Fix crasher: open in 'wb' mode.
469 except IOError:
470 g.warning("can not open", fileName)
471 return
472 for p in p.self_and_subtree(copy=False):
473 s = p.moreHead(firstLevel) + nl
474 s = g.toEncodedString(s, encoding=self.encoding, reportErrors=True)
475 theFile.write(s)
476 s = p.moreBody() + nl # Inserts escapes.
477 if s.strip():
478 s = g.toEncodedString(s, self.encoding, reportErrors=True)
479 theFile.write(s)
480 theFile.close()
481 #@+node:ekr.20031218072017.1148: *4* ic.outlineToWeb
482 def outlineToWeb(self, fileName, webType):
483 c = self.c
484 nl = self.output_newline
485 current = c.p
486 if not current:
487 return
488 self.setEncoding()
489 self.webType = webType
490 try:
491 theFile = open(fileName, 'w')
492 except IOError:
493 g.warning("can not open", fileName)
494 return
495 self.treeType = "@file"
496 # Set self.treeType to @root if p or an ancestor is an @root node.
497 for p in current.parents():
498 flag, junk = g.is_special(p.b, "@root")
499 if flag:
500 self.treeType = "@root"
501 break
502 for p in current.self_and_subtree(copy=False):
503 s = self.convertVnodeToWeb(p)
504 if s:
505 theFile.write(s)
506 if s[-1] != '\n':
507 theFile.write(nl)
508 theFile.close()
509 #@+node:ekr.20031218072017.3300: *4* ic.removeSentinelsCommand
510 def removeSentinelsCommand(self, paths, toString=False):
511 c = self.c
512 self.setEncoding()
513 for fileName in paths:
514 g.setGlobalOpenDir(fileName)
515 path, self.fileName = g.os_path_split(fileName)
516 s, e = g.readFileIntoString(fileName, self.encoding)
517 if s is None:
518 return None
519 if e:
520 self.encoding = e
521 #@+<< set delims from the header line >>
522 #@+node:ekr.20031218072017.3302: *5* << set delims from the header line >>
523 # Skip any non @+leo lines.
524 i = 0
525 while i < len(s) and g.find_on_line(s, i, "@+leo") == -1:
526 i = g.skip_line(s, i)
527 # Get the comment delims from the @+leo sentinel line.
528 at = self.c.atFileCommands
529 j = g.skip_line(s, i)
530 line = s[i:j]
531 valid, junk, start_delim, end_delim, junk = at.parseLeoSentinel(line)
532 if not valid:
533 if not toString:
534 g.es("invalid @+leo sentinel in", fileName)
535 return None
536 if end_delim:
537 line_delim = None
538 else:
539 line_delim, start_delim = start_delim, None
540 #@-<< set delims from the header line >>
541 s = self.removeSentinelLines(s, line_delim, start_delim, end_delim)
542 ext = c.config.remove_sentinels_extension
543 if not ext:
544 ext = ".txt"
545 if ext[0] == '.':
546 newFileName = g.os_path_finalize_join(path, fileName + ext) # 1341
547 else:
548 head, ext2 = g.os_path_splitext(fileName)
549 newFileName = g.os_path_finalize_join(path, head + ext + ext2) # 1341
550 if toString:
551 return s
552 #@+<< Write s into newFileName >>
553 #@+node:ekr.20031218072017.1149: *5* << Write s into newFileName >> (remove-sentinels)
554 # Remove sentinels command.
555 try:
556 with open(newFileName, 'w') as theFile:
557 theFile.write(s)
558 if not g.unitTesting:
559 g.es("created:", newFileName)
560 except Exception:
561 g.es("exception creating:", newFileName)
562 g.print_exception()
563 #@-<< Write s into newFileName >>
564 return None
565 #@+node:ekr.20031218072017.3303: *4* ic.removeSentinelLines
566 # This does not handle @nonl properly, but that no longer matters.
568 def removeSentinelLines(self, s, line_delim, start_delim, unused_end_delim):
569 """Properly remove all sentinle lines in s."""
570 delim = (line_delim or start_delim or '') + '@'
571 verbatim = delim + 'verbatim'
572 verbatimFlag = False
573 result = []
574 for line in g.splitLines(s):
575 i = g.skip_ws(line, 0)
576 if not verbatimFlag and g.match(line, i, delim):
577 if g.match(line, i, verbatim):
578 # Force the next line to be in the result.
579 verbatimFlag = True
580 else:
581 result.append(line)
582 verbatimFlag = False
583 return ''.join(result)
584 #@+node:ekr.20031218072017.1464: *4* ic.weave
585 def weave(self, filename):
586 p = self.c.p
587 nl = self.output_newline
588 if not p:
589 return
590 self.setEncoding()
591 try:
592 with open(filename, 'w', encoding=self.encoding) as f:
593 for p in p.self_and_subtree():
594 s = p.b
595 s2 = s.strip()
596 if s2:
597 f.write("-" * 60)
598 f.write(nl)
599 #@+<< write the context of p to f >>
600 #@+node:ekr.20031218072017.1465: *5* << write the context of p to f >> (weave)
601 # write the headlines of p, p's parent and p's grandparent.
602 context = []
603 p2 = p.copy()
604 i = 0
605 while i < 3:
606 i += 1
607 if not p2:
608 break
609 context.append(p2.h)
610 p2.moveToParent()
611 context.reverse()
612 indent = ""
613 for line in context:
614 f.write(indent)
615 indent += '\t'
616 f.write(line)
617 f.write(nl)
618 #@-<< write the context of p to f >>
619 f.write("-" * 60)
620 f.write(nl)
621 f.write(s.rstrip() + nl)
622 except Exception:
623 g.es("exception opening:", filename)
624 g.print_exception()
625 #@+node:ekr.20031218072017.3209: *3* ic.Import
626 #@+node:ekr.20031218072017.3210: *4* ic.createOutline & helpers
627 def createOutline(self, parent, ext=None, s=None):
628 """
629 Create an outline by importing a file, reading the file with the
630 given encoding if string s is None.
632 ext, The file extension to be used, or None.
633 fileName: A string or None. The name of the file to be read.
634 parent: The parent position of the created outline.
635 s: A string or None. The file's contents.
636 """
637 c = self.c
638 p = parent.copy()
639 self.treeType = '@file' # Fix #352.
640 fileName = g.fullPath(c, parent)
641 if g.is_binary_external_file(fileName):
642 return self.import_binary_file(fileName, parent)
643 # Init ivars.
644 self.setEncoding(
645 p=parent,
646 default=c.config.default_at_auto_file_encoding,
647 )
648 ext, s = self.init_import(ext, fileName, s)
649 if s is None:
650 return None
651 # The so-called scanning func is a callback. It must have a c argument.
652 func = self.dispatch(ext, p)
653 # Call the scanning function.
654 if g.unitTesting:
655 assert func or ext in ('.txt', '.w', '.xxx'), (repr(func), ext, p.h)
656 if func and not c.config.getBool('suppress-import-parsing', default=False):
657 s = g.toUnicode(s, encoding=self.encoding)
658 s = s.replace('\r', '')
659 # func is a factory that instantiates the importer class.
660 ok = func(c=c, parent=p, s=s)
661 else:
662 # Just copy the file to the parent node.
663 s = g.toUnicode(s, encoding=self.encoding)
664 s = s.replace('\r', '')
665 ok = self.scanUnknownFileType(s, p, ext)
666 if g.unitTesting:
667 return p if ok else None
668 # #488894: unsettling dialog when saving Leo file
669 # #889175: Remember the full fileName.
670 c.atFileCommands.rememberReadPath(fileName, p)
671 p.contract()
672 w = c.frame.body.wrapper
673 w.setInsertPoint(0)
674 w.seeInsertPoint()
675 return p
676 #@+node:ekr.20140724064952.18038: *5* ic.dispatch & helpers
677 def dispatch(self, ext, p):
678 """Return the correct scanner function for p, an @auto node."""
679 # Match the @auto type first, then the file extension.
680 c = self.c
681 return g.app.scanner_for_at_auto(c, p) or g.app.scanner_for_ext(c, ext)
682 #@+node:ekr.20170405191106.1: *5* ic.import_binary_file
683 def import_binary_file(self, fileName, parent):
685 # Fix bug 1185409 importing binary files puts binary content in body editor.
686 # Create an @url node.
687 c = self.c
688 if parent:
689 p = parent.insertAsLastChild()
690 else:
691 p = c.lastTopLevel().insertAfter()
692 p.h = f"@url file://{fileName}"
693 return p
694 #@+node:ekr.20140724175458.18052: *5* ic.init_import
695 def init_import(self, ext, fileName, s):
696 """
697 Init ivars imports and read the file into s.
698 Return ext, s.
699 """
700 junk, self.fileName = g.os_path_split(fileName)
701 self.methodName, self.fileType = g.os_path_splitext(self.fileName)
702 if not ext:
703 ext = self.fileType
704 ext = ext.lower()
705 if not s:
706 # Set the kind for error messages in readFileIntoString.
707 s, e = g.readFileIntoString(fileName, encoding=self.encoding)
708 if s is None:
709 return None, None
710 if e:
711 self.encoding = e
712 return ext, s
713 #@+node:ekr.20070713075352: *5* ic.scanUnknownFileType & helper
714 def scanUnknownFileType(self, s, p, ext):
715 """Scan the text of an unknown file type."""
716 body = ''
717 if ext in ('.html', '.htm'):
718 body += '@language html\n'
719 elif ext in ('.txt', '.text'):
720 body += '@nocolor\n'
721 else:
722 language = self.languageForExtension(ext)
723 if language:
724 body += f"@language {language}\n"
725 self.setBodyString(p, body + s)
726 for p in p.self_and_subtree():
727 p.clearDirty()
728 return True
729 #@+node:ekr.20080811174246.1: *6* ic.languageForExtension
730 def languageForExtension(self, ext):
731 """Return the language corresponding to the extension ext."""
732 unknown = 'unknown_language'
733 if ext.startswith('.'):
734 ext = ext[1:]
735 if ext:
736 z = g.app.extra_extension_dict.get(ext)
737 if z not in (None, 'none', 'None'):
738 language = z
739 else:
740 language = g.app.extension_dict.get(ext)
741 if language in (None, 'none', 'None'):
742 language = unknown
743 else:
744 language = unknown
745 # Return the language even if there is no colorizer mode for it.
746 return language
747 #@+node:ekr.20070806111212: *4* ic.readAtAutoNodes
748 def readAtAutoNodes(self):
749 c, p = self.c, self.c.p
750 after = p.nodeAfterTree()
751 found = False
752 while p and p != after:
753 if p.isAtAutoNode():
754 if p.isAtIgnoreNode():
755 g.warning('ignoring', p.h)
756 p.moveToThreadNext()
757 else:
758 c.atFileCommands.readOneAtAutoNode(p)
759 found = True
760 p.moveToNodeAfterTree()
761 else:
762 p.moveToThreadNext()
763 if not g.unitTesting:
764 message = 'finished' if found else 'no @auto nodes in the selected tree'
765 g.blue(message)
766 c.redraw()
767 #@+node:ekr.20031218072017.1810: *4* ic.importDerivedFiles
768 def importDerivedFiles(self, parent=None, paths=None, command='Import'):
769 """
770 Import one or more external files.
771 This is not a command. It must *not* have an event arg.
772 command is None when importing from the command line.
773 """
774 at, c, u = self.c.atFileCommands, self.c, self.c.undoer
775 current = c.p or c.rootPosition()
776 self.tab_width = c.getTabWidth(current)
777 if not paths:
778 return None
779 # Initial open from command line is not undoable.
780 if command:
781 u.beforeChangeGroup(current, command)
782 for fileName in paths:
783 fileName = fileName.replace('\\', '/') # 2011/10/09.
784 g.setGlobalOpenDir(fileName)
785 isThin = at.scanHeaderForThin(fileName)
786 if command:
787 undoData = u.beforeInsertNode(parent)
788 p = parent.insertAfter()
789 if isThin:
790 # Create @file node, not a deprecated @thin node.
791 p.initHeadString("@file " + fileName)
792 at.read(p)
793 else:
794 p.initHeadString("Imported @file " + fileName)
795 at.read(p)
796 p.contract()
797 p.setDirty() # 2011/10/09: tell why the file is dirty!
798 if command:
799 u.afterInsertNode(p, command, undoData)
800 current.expand()
801 c.setChanged()
802 if command:
803 u.afterChangeGroup(p, command)
804 c.redraw(current)
805 return p
806 #@+node:ekr.20031218072017.3212: *4* ic.importFilesCommand
807 def importFilesCommand(self,
808 files=None,
809 parent=None,
810 shortFn=False,
811 treeType=None,
812 verbose=True, # Legacy value.
813 ):
814 # Not a command. It must *not* have an event arg.
815 c, u = self.c, self.c.undoer
816 if not c or not c.p or not files:
817 return
818 self.tab_width = c.getTabWidth(c.p)
819 self.treeType = treeType or '@file'
820 self.verbose = verbose
821 if not parent:
822 g.trace('===== no parent', g.callers())
823 return
824 for fn in files or []:
825 # Report exceptions here, not in the caller.
826 try:
827 g.setGlobalOpenDir(fn)
828 # Leo 5.6: Handle undo here, not in createOutline.
829 undoData = u.beforeInsertNode(parent)
830 p = parent.insertAsLastChild()
831 p.h = f"{treeType} {fn}"
832 u.afterInsertNode(p, 'Import', undoData)
833 p = self.createOutline(parent=p)
834 if p: # createOutline may fail.
835 if self.verbose and not g.unitTesting:
836 g.blue("imported", g.shortFileName(fn) if shortFn else fn)
837 p.contract()
838 p.setDirty()
839 c.setChanged()
840 except Exception:
841 g.es_print('Exception importing', fn)
842 g.es_exception()
843 c.validateOutline()
844 parent.expand()
845 #@+node:ekr.20160503125237.1: *4* ic.importFreeMind
846 def importFreeMind(self, files):
847 """
848 Import a list of .mm.html files exported from FreeMind:
849 http://freemind.sourceforge.net/wiki/index.php/Main_Page
850 """
851 FreeMindImporter(self.c).import_files(files)
852 #@+node:ekr.20160503125219.1: *4* ic.importMindMap
853 def importMindMap(self, files):
854 """
855 Import a list of .csv files exported from MindJet:
856 https://www.mindjet.com/
857 """
858 MindMapImporter(self.c).import_files(files)
859 #@+node:ekr.20031218072017.3224: *4* ic.importWebCommand & helpers
860 def importWebCommand(self, files, webType):
861 c, current = self.c, self.c.p
862 if current is None:
863 return
864 if not files:
865 return
866 self.tab_width = c.getTabWidth(current) # New in 4.3.
867 self.webType = webType
868 for fileName in files:
869 g.setGlobalOpenDir(fileName)
870 p = self.createOutlineFromWeb(fileName, current)
871 p.contract()
872 p.setDirty()
873 c.setChanged()
874 c.redraw(current)
875 #@+node:ekr.20031218072017.3225: *5* createOutlineFromWeb
876 def createOutlineFromWeb(self, path, parent):
877 c = self.c
878 u = c.undoer
879 junk, fileName = g.os_path_split(path)
880 undoData = u.beforeInsertNode(parent)
881 # Create the top-level headline.
882 p = parent.insertAsLastChild()
883 p.initHeadString(fileName)
884 if self.webType == "cweb":
885 self.setBodyString(p, "@ignore\n@language cweb")
886 # Scan the file, creating one section for each function definition.
887 self.scanWebFile(path, p)
888 u.afterInsertNode(p, 'Import', undoData)
889 return p
890 #@+node:ekr.20031218072017.3227: *5* findFunctionDef
891 def findFunctionDef(self, s, i):
892 # Look at the next non-blank line for a function name.
893 i = g.skip_ws_and_nl(s, i)
894 k = g.skip_line(s, i)
895 name = None
896 while i < k:
897 if g.is_c_id(s[i]):
898 j = i
899 i = g.skip_c_id(s, i)
900 name = s[j:i]
901 elif s[i] == '(':
902 if name:
903 return name
904 break
905 else:
906 i += 1
907 return None
908 #@+node:ekr.20031218072017.3228: *5* scanBodyForHeadline
909 #@+at This method returns the proper headline text.
910 # 1. If s contains a section def, return the section ref.
911 # 2. cweb only: if s contains @c, return the function name following the @c.
912 # 3. cweb only: if s contains @d name, returns @d name.
913 # 4. Otherwise, returns "@"
914 #@@c
916 def scanBodyForHeadline(self, s):
917 if self.webType == "cweb":
918 #@+<< scan cweb body for headline >>
919 #@+node:ekr.20031218072017.3229: *6* << scan cweb body for headline >>
920 i = 0
921 while i < len(s):
922 i = g.skip_ws_and_nl(s, i)
923 # Allow constructs such as @ @c, or @ @<.
924 if self.isDocStart(s, i):
925 i += 2
926 i = g.skip_ws(s, i)
927 if g.match(s, i, "@d") or g.match(s, i, "@f"):
928 # Look for a macro name.
929 directive = s[i : i + 2]
930 i = g.skip_ws(s, i + 2) # skip the @d or @f
931 if i < len(s) and g.is_c_id(s[i]):
932 j = i
933 g.skip_c_id(s, i)
934 return s[j:i]
935 return directive
936 if g.match(s, i, "@c") or g.match(s, i, "@p"):
937 # Look for a function def.
938 name = self.findFunctionDef(s, i + 2)
939 return name if name else "outer function"
940 if g.match(s, i, "@<"):
941 # Look for a section def.
942 # A small bug: the section def must end on this line.
943 j = i
944 k = g.find_on_line(s, i, "@>")
945 if k > -1 and (g.match(s, k + 2, "+=") or g.match(s, k + 2, "=")):
946 return s[j : k + 2] # return the section ref.
947 i = g.skip_line(s, i)
948 #@-<< scan cweb body for headline >>
949 else:
950 #@+<< scan noweb body for headline >>
951 #@+node:ekr.20031218072017.3230: *6* << scan noweb body for headline >>
952 i = 0
953 while i < len(s):
954 i = g.skip_ws_and_nl(s, i)
955 if g.match(s, i, "<<"):
956 k = g.find_on_line(s, i, ">>=")
957 if k > -1:
958 ref = s[i : k + 2]
959 name = s[i + 2 : k].strip()
960 if name != "@others":
961 return ref
962 else:
963 name = self.findFunctionDef(s, i)
964 if name:
965 return name
966 i = g.skip_line(s, i)
967 #@-<< scan noweb body for headline >>
968 return "@" # default.
969 #@+node:ekr.20031218072017.3231: *5* scanWebFile (handles limbo)
970 def scanWebFile(self, fileName, parent):
971 theType = self.webType
972 lb = "@<" if theType == "cweb" else "<<"
973 rb = "@>" if theType == "cweb" else ">>"
974 s, e = g.readFileIntoString(fileName)
975 if s is None:
976 return
977 #@+<< Create a symbol table of all section names >>
978 #@+node:ekr.20031218072017.3232: *6* << Create a symbol table of all section names >>
979 i = 0
980 self.web_st = []
981 while i < len(s):
982 progress = i
983 i = g.skip_ws_and_nl(s, i)
984 if self.isDocStart(s, i):
985 if theType == "cweb":
986 i += 2
987 else:
988 i = g.skip_line(s, i)
989 elif theType == "cweb" and g.match(s, i, "@@"):
990 i += 2
991 elif g.match(s, i, lb):
992 i += 2
993 j = i
994 k = g.find_on_line(s, j, rb)
995 if k > -1:
996 self.cstEnter(s[j:k])
997 else:
998 i += 1
999 assert i > progress
1000 #@-<< Create a symbol table of all section names >>
1001 #@+<< Create nodes for limbo text and the root section >>
1002 #@+node:ekr.20031218072017.3233: *6* << Create nodes for limbo text and the root section >>
1003 i = 0
1004 while i < len(s):
1005 progress = i
1006 i = g.skip_ws_and_nl(s, i)
1007 if self.isModuleStart(s, i) or g.match(s, i, lb):
1008 break
1009 else: i = g.skip_line(s, i)
1010 assert i > progress
1011 j = g.skip_ws(s, 0)
1012 if j < i:
1013 self.createHeadline(parent, "@ " + s[j:i], "Limbo")
1014 j = i
1015 if g.match(s, i, lb):
1016 while i < len(s):
1017 progress = i
1018 i = g.skip_ws_and_nl(s, i)
1019 if self.isModuleStart(s, i):
1020 break
1021 else: i = g.skip_line(s, i)
1022 assert i > progress
1023 self.createHeadline(parent, s[j:i], g.angleBrackets(" @ "))
1025 #@-<< Create nodes for limbo text and the root section >>
1026 while i < len(s):
1027 outer_progress = i
1028 #@+<< Create a node for the next module >>
1029 #@+node:ekr.20031218072017.3234: *6* << Create a node for the next module >>
1030 if theType == "cweb":
1031 assert self.isModuleStart(s, i)
1032 start = i
1033 if self.isDocStart(s, i):
1034 i += 2
1035 while i < len(s):
1036 progress = i
1037 i = g.skip_ws_and_nl(s, i)
1038 if self.isModuleStart(s, i):
1039 break
1040 else:
1041 i = g.skip_line(s, i)
1042 assert i > progress
1043 #@+<< Handle cweb @d, @f, @c and @p directives >>
1044 #@+node:ekr.20031218072017.3235: *7* << Handle cweb @d, @f, @c and @p directives >>
1045 if g.match(s, i, "@d") or g.match(s, i, "@f"):
1046 i += 2
1047 i = g.skip_line(s, i)
1048 # Place all @d and @f directives in the same node.
1049 while i < len(s):
1050 progress = i
1051 i = g.skip_ws_and_nl(s, i)
1052 if g.match(s, i, "@d") or g.match(s, i, "@f"):
1053 i = g.skip_line(s, i)
1054 else:
1055 break
1056 assert i > progress
1057 i = g.skip_ws_and_nl(s, i)
1058 while i < len(s) and not self.isModuleStart(s, i):
1059 progress = i
1060 i = g.skip_line(s, i)
1061 i = g.skip_ws_and_nl(s, i)
1062 assert i > progress
1063 if g.match(s, i, "@c") or g.match(s, i, "@p"):
1064 i += 2
1065 while i < len(s):
1066 progress = i
1067 i = g.skip_line(s, i)
1068 i = g.skip_ws_and_nl(s, i)
1069 if self.isModuleStart(s, i):
1070 break
1071 assert i > progress
1072 #@-<< Handle cweb @d, @f, @c and @p directives >>
1073 else:
1074 assert self.isDocStart(s, i)
1075 start = i
1076 i = g.skip_line(s, i)
1077 while i < len(s):
1078 progress = i
1079 i = g.skip_ws_and_nl(s, i)
1080 if self.isDocStart(s, i):
1081 break
1082 else:
1083 i = g.skip_line(s, i)
1084 assert i > progress
1085 body = s[start:i]
1086 body = self.massageWebBody(body)
1087 headline = self.scanBodyForHeadline(body)
1088 self.createHeadline(parent, body, headline)
1089 #@-<< Create a node for the next module >>
1090 assert i > outer_progress
1091 #@+node:ekr.20031218072017.3236: *5* Symbol table
1092 #@+node:ekr.20031218072017.3237: *6* cstCanonicalize
1093 # We canonicalize strings before looking them up,
1094 # but strings are entered in the form they are first encountered.
1096 def cstCanonicalize(self, s, lower=True):
1097 if lower:
1098 s = s.lower()
1099 s = s.replace("\t", " ").replace("\r", "")
1100 s = s.replace("\n", " ").replace(" ", " ")
1101 return s.strip()
1102 #@+node:ekr.20031218072017.3238: *6* cstDump
1103 def cstDump(self):
1104 s = "Web Symbol Table...\n\n"
1105 for name in sorted(self.web_st):
1106 s += name + "\n"
1107 return s
1108 #@+node:ekr.20031218072017.3239: *6* cstEnter
1109 # We only enter the section name into the symbol table if the ... convention is not used.
1111 def cstEnter(self, s):
1112 # Don't enter names that end in "..."
1113 s = s.rstrip()
1114 if s.endswith("..."):
1115 return
1116 # Put the section name in the symbol table, retaining capitalization.
1117 lower = self.cstCanonicalize(s, True) # do lower
1118 upper = self.cstCanonicalize(s, False) # don't lower.
1119 for name in self.web_st:
1120 if name.lower() == lower:
1121 return
1122 self.web_st.append(upper)
1123 #@+node:ekr.20031218072017.3240: *6* cstLookup
1124 # This method returns a string if the indicated string is a prefix of an entry in the web_st.
1126 def cstLookup(self, target):
1127 # Do nothing if the ... convention is not used.
1128 target = target.strip()
1129 if not target.endswith("..."):
1130 return target
1131 # Canonicalize the target name, and remove the trailing "..."
1132 ctarget = target[:-3]
1133 ctarget = self.cstCanonicalize(ctarget).strip()
1134 found = False
1135 result = target
1136 for s in self.web_st:
1137 cs = self.cstCanonicalize(s)
1138 if cs[: len(ctarget)] == ctarget:
1139 if found:
1140 g.es('', f"****** {target}", 'is also a prefix of', s)
1141 else:
1142 found = True
1143 result = s
1144 # g.es("replacing",target,"with",s)
1145 return result
1146 #@+node:ekr.20140531104908.18833: *3* ic.parse_body
1147 def parse_body(self, p):
1148 """
1149 Parse p.b as source code, creating a tree of descendant nodes.
1150 This is essentially an import of p.b.
1151 """
1152 if not p:
1153 return
1154 c, d, ic = self.c, g.app.language_extension_dict, self
1155 if p.hasChildren():
1156 g.es_print('can not run parse-body: node has children:', p.h)
1157 return
1158 language = g.scanForAtLanguage(c, p)
1159 self.treeType = '@file'
1160 ext = '.' + d.get(language)
1161 parser = g.app.classDispatchDict.get(ext)
1162 # Fix bug 151: parse-body creates "None declarations"
1163 if p.isAnyAtFileNode():
1164 fn = p.anyAtFileNodeName()
1165 ic.methodName, ic.fileType = g.os_path_splitext(fn)
1166 else:
1167 fileType = d.get(language, 'py')
1168 ic.methodName, ic.fileType = p.h, fileType
1169 if not parser:
1170 g.es_print(f"parse-body: no parser for @language {language or 'None'}")
1171 return
1172 bunch = c.undoer.beforeChangeTree(p)
1173 s = p.b
1174 p.b = ''
1175 try:
1176 parser(c, s, p) # 2357.
1177 c.undoer.afterChangeTree(p, 'parse-body', bunch)
1178 p.expand()
1179 c.selectPosition(p)
1180 c.redraw()
1181 except Exception:
1182 g.es_exception()
1183 p.b = s
1184 #@+node:ekr.20031218072017.3305: *3* ic.Utilities
1185 #@+node:ekr.20090122201952.4: *4* ic.appendStringToBody & setBodyString (leoImport)
1186 def appendStringToBody(self, p, s):
1187 """Similar to c.appendStringToBody,
1188 but does not recolor the text or redraw the screen."""
1189 if s:
1190 p.b = p.b + g.toUnicode(s, self.encoding)
1192 def setBodyString(self, p, s):
1193 """
1194 Similar to c.setBodyString, but does not recolor the text or
1195 redraw the screen.
1196 """
1197 c, v = self.c, p.v
1198 if not c or not p:
1199 return
1200 s = g.toUnicode(s, self.encoding)
1201 if c.p and p.v == c.p.v:
1202 w = c.frame.body.wrapper
1203 i = len(s)
1204 w.setAllText(s)
1205 w.setSelectionRange(i, i, insert=i)
1206 # Keep the body text up-to-date.
1207 if v.b != s:
1208 v.setBodyString(s)
1209 v.setSelection(0, 0)
1210 p.setDirty()
1211 if not c.isChanged():
1212 c.setChanged()
1213 #@+node:ekr.20031218072017.3306: *4* ic.createHeadline
1214 def createHeadline(self, parent, body, headline):
1215 """Create a new VNode as the last child of parent position."""
1216 p = parent.insertAsLastChild()
1217 p.initHeadString(headline)
1218 if body:
1219 self.setBodyString(p, body)
1220 return p
1221 #@+node:ekr.20031218072017.3307: *4* ic.error
1222 def error(self, s):
1223 g.es('', s)
1224 #@+node:ekr.20031218072017.3309: *4* ic.isDocStart & isModuleStart
1225 # The start of a document part or module in a noweb or cweb file.
1226 # Exporters may have to test for @doc as well.
1228 def isDocStart(self, s, i):
1229 if not g.match(s, i, "@"):
1230 return False
1231 j = g.skip_ws(s, i + 1)
1232 if g.match(s, j, "%defs"):
1233 return False
1234 if self.webType == "cweb" and g.match(s, i, "@*"):
1235 return True
1236 return g.match(s, i, "@ ") or g.match(s, i, "@\t") or g.match(s, i, "@\n")
1238 def isModuleStart(self, s, i):
1239 if self.isDocStart(s, i):
1240 return True
1241 return self.webType == "cweb" and (
1242 g.match(s, i, "@c") or g.match(s, i, "@p") or
1243 g.match(s, i, "@d") or g.match(s, i, "@f"))
1244 #@+node:ekr.20031218072017.3312: *4* ic.massageWebBody
1245 def massageWebBody(self, s):
1246 theType = self.webType
1247 lb = "@<" if theType == "cweb" else "<<"
1248 rb = "@>" if theType == "cweb" else ">>"
1249 #@+<< Remove most newlines from @space and @* sections >>
1250 #@+node:ekr.20031218072017.3313: *5* << Remove most newlines from @space and @* sections >>
1251 i = 0
1252 while i < len(s):
1253 progress = i
1254 i = g.skip_ws_and_nl(s, i)
1255 if self.isDocStart(s, i):
1256 # Scan to end of the doc part.
1257 if g.match(s, i, "@ %def"):
1258 # Don't remove the newline following %def
1259 i = g.skip_line(s, i)
1260 start = end = i
1261 else:
1262 start = end = i
1263 i += 2
1264 while i < len(s):
1265 progress2 = i
1266 i = g.skip_ws_and_nl(s, i)
1267 if self.isModuleStart(s, i) or g.match(s, i, lb):
1268 end = i
1269 break
1270 elif theType == "cweb":
1271 i += 1
1272 else:
1273 i = g.skip_to_end_of_line(s, i)
1274 assert i > progress2
1275 # Remove newlines from start to end.
1276 doc = s[start:end]
1277 doc = doc.replace("\n", " ")
1278 doc = doc.replace("\r", "")
1279 doc = doc.strip()
1280 if doc:
1281 if doc == "@":
1282 doc = "@ " if self.webType == "cweb" else "@\n"
1283 else:
1284 doc += "\n\n"
1285 s = s[:start] + doc + s[end:]
1286 i = start + len(doc)
1287 else: i = g.skip_line(s, i)
1288 assert i > progress
1289 #@-<< Remove most newlines from @space and @* sections >>
1290 #@+<< Replace abbreviated names with full names >>
1291 #@+node:ekr.20031218072017.3314: *5* << Replace abbreviated names with full names >>
1292 i = 0
1293 while i < len(s):
1294 progress = i
1295 if g.match(s, i, lb):
1296 i += 2
1297 j = i
1298 k = g.find_on_line(s, j, rb)
1299 if k > -1:
1300 name = s[j:k]
1301 name2 = self.cstLookup(name)
1302 if name != name2:
1303 # Replace name by name2 in s.
1304 s = s[:j] + name2 + s[k:]
1305 i = j + len(name2)
1306 i = g.skip_line(s, i)
1307 assert i > progress
1308 #@-<< Replace abbreviated names with full names >>
1309 s = s.rstrip()
1310 return s
1311 #@+node:ekr.20031218072017.1463: *4* ic.setEncoding
1312 def setEncoding(self, p=None, default=None):
1313 c = self.c
1314 encoding = g.getEncodingAt(p or c.p) or default
1315 if encoding and g.isValidEncoding(encoding):
1316 self.encoding = encoding
1317 elif default:
1318 self.encoding = default
1319 else:
1320 self.encoding = 'utf-8'
1321 #@-others
1322#@+node:ekr.20160503144404.1: ** class MindMapImporter
1323class MindMapImporter:
1324 """Mind Map Importer class."""
1326 def __init__(self, c):
1327 """ctor for MindMapImporter class."""
1328 self.c = c
1329 #@+others
1330 #@+node:ekr.20160503130209.1: *3* mindmap.create_outline
1331 def create_outline(self, path):
1332 c = self.c
1333 junk, fileName = g.os_path_split(path)
1334 undoData = c.undoer.beforeInsertNode(c.p)
1335 # Create the top-level headline.
1336 p = c.lastTopLevel().insertAfter()
1337 fn = g.shortFileName(path).strip()
1338 if fn.endswith('.csv'):
1339 fn = fn[:-4]
1340 p.h = fn
1341 try:
1342 self.scan(path, p)
1343 except Exception:
1344 g.es_print('Invalid MindJet file:', fn)
1345 c.undoer.afterInsertNode(p, 'Import', undoData)
1346 return p
1347 #@+node:ekr.20160503144647.1: *3* mindmap.import_files
1348 def import_files(self, files):
1349 """Import a list of MindMap (.csv) files."""
1350 c = self.c
1351 if files:
1352 self.tab_width = c.getTabWidth(c.p)
1353 for fileName in files:
1354 g.setGlobalOpenDir(fileName)
1355 p = self.create_outline(fileName)
1356 p.contract()
1357 p.setDirty()
1358 c.setChanged()
1359 c.redraw(p)
1360 #@+node:ekr.20160504043243.1: *3* mindmap.prompt_for_files
1361 def prompt_for_files(self):
1362 """Prompt for a list of MindJet (.csv) files and import them."""
1363 c = self.c
1364 types = [
1365 ("MindJet files", "*.csv"),
1366 ("All files", "*"),
1367 ]
1368 names = g.app.gui.runOpenFileDialog(c,
1369 title="Import MindJet File",
1370 filetypes=types,
1371 defaultextension=".csv",
1372 multiple=True)
1373 c.bringToFront()
1374 if names:
1375 g.chdir(names[0])
1376 self.import_files(names)
1377 #@+node:ekr.20160503130256.1: *3* mindmap.scan & helpers
1378 def scan(self, path, target):
1379 """Create an outline from a MindMap (.csv) file."""
1380 c = self.c
1381 f = open(path)
1382 reader = csv.reader(f)
1383 max_chars_in_header = 80
1384 n1 = n = target.level()
1385 p = target.copy()
1386 for row in list(reader)[1:]:
1387 new_level = self.csv_level(row) + n1
1388 self.csv_string(row)
1389 if new_level > n:
1390 p = p.insertAsLastChild().copy()
1391 p.b = self.csv_string(row)
1392 n = n + 1
1393 elif new_level == n:
1394 p = p.insertAfter().copy()
1395 p.b = self.csv_string(row)
1396 elif new_level < n:
1397 for item in p.parents():
1398 if item.level() == new_level - 1:
1399 p = item.copy()
1400 break
1401 p = p.insertAsLastChild().copy()
1402 p.b = self.csv_string(row)
1403 n = p.level()
1404 for p in target.unique_subtree():
1405 if len(p.b.splitlines()) == 1:
1406 if len(p.b.splitlines()[0]) < max_chars_in_header:
1407 p.h = p.b.splitlines()[0]
1408 p.b = ""
1409 else:
1410 p.h = "@node_with_long_text"
1411 else:
1412 p.h = "@node_with_long_text"
1413 c.redraw()
1414 f.close()
1415 #@+node:ekr.20160503130810.4: *4* mindmap.csv_level
1416 def csv_level(self, row):
1417 """Return the level of the given row."""
1418 count = 0
1419 while count <= len(row):
1420 if row[count]:
1421 return count + 1
1422 count = count + 1
1423 return -1
1424 #@+node:ekr.20160503130810.5: *4* mindmap.csv_string
1425 def csv_string(self, row):
1426 """Return the string for the given csv row."""
1427 count = 0
1428 while count <= len(row):
1429 if row[count]:
1430 return row[count]
1431 count = count + 1
1432 return None
1433 #@-others
1434#@+node:ekr.20161006100941.1: ** class MORE_Importer
1435class MORE_Importer:
1436 """Class to import MORE files."""
1438 def __init__(self, c):
1439 """ctor for MORE_Importer class."""
1440 self.c = c
1441 #@+others
1442 #@+node:ekr.20161006101111.1: *3* MORE.prompt_for_files
1443 def prompt_for_files(self):
1444 """Prompt for a list of MORE files and import them."""
1445 c = self.c
1446 types = [
1447 ("All files", "*"),
1448 ]
1449 names = g.app.gui.runOpenFileDialog(c,
1450 title="Import MORE Files",
1451 filetypes=types,
1452 # defaultextension=".txt",
1453 multiple=True)
1454 c.bringToFront()
1455 if names:
1456 g.chdir(names[0])
1457 self.import_files(names)
1458 #@+node:ekr.20161006101218.1: *3* MORE.import_files
1459 def import_files(self, files):
1460 """Import a list of MORE (.csv) files."""
1461 c = self.c
1462 if files:
1463 changed = False
1464 self.tab_width = c.getTabWidth(c.p)
1465 for fileName in files:
1466 g.setGlobalOpenDir(fileName)
1467 p = self.import_file(fileName)
1468 if p:
1469 p.contract()
1470 p.setDirty()
1471 c.setChanged()
1472 changed = True
1473 if changed:
1474 c.redraw(p)
1475 #@+node:ekr.20161006101347.1: *3* MORE.import_file
1476 def import_file(self, fileName): # Not a command, so no event arg.
1477 c = self.c
1478 u = c.undoer
1479 ic = c.importCommands
1480 if not c.p:
1481 return None
1482 ic.setEncoding()
1483 g.setGlobalOpenDir(fileName)
1484 s, e = g.readFileIntoString(fileName)
1485 if s is None:
1486 return None
1487 s = s.replace('\r', '') # Fixes bug 626101.
1488 lines = g.splitLines(s)
1489 # Convert the string to an outline and insert it after the current node.
1490 if self.check_lines(lines):
1491 last = c.lastTopLevel()
1492 undoData = u.beforeInsertNode(c.p)
1493 root = last.insertAfter()
1494 root.h = fileName
1495 p = self.import_lines(lines, root)
1496 if p:
1497 c.endEditing()
1498 c.validateOutline()
1499 p.setDirty()
1500 c.setChanged()
1501 u.afterInsertNode(root, 'Import MORE File', undoData)
1502 c.selectPosition(root)
1503 c.redraw()
1504 return root
1505 if not g.unitTesting:
1506 g.es("not a valid MORE file", fileName)
1507 return None
1508 #@+node:ekr.20031218072017.3215: *3* MORE.import_lines
1509 def import_lines(self, strings, first_p):
1510 c = self.c
1511 if not strings:
1512 return None
1513 if not self.check_lines(strings):
1514 return None
1515 firstLevel, junk = self.headlineLevel(strings[0])
1516 lastLevel = -1
1517 theRoot = last_p = None
1518 index = 0
1519 while index < len(strings):
1520 progress = index
1521 s = strings[index]
1522 level, junk = self.headlineLevel(s)
1523 level -= firstLevel
1524 if level >= 0:
1525 #@+<< Link a new position p into the outline >>
1526 #@+node:ekr.20031218072017.3216: *4* << Link a new position p into the outline >>
1527 assert level >= 0
1528 if not last_p:
1529 theRoot = p = first_p.insertAsLastChild() # 2016/10/06.
1530 elif level == lastLevel:
1531 p = last_p.insertAfter()
1532 elif level == lastLevel + 1:
1533 p = last_p.insertAsNthChild(0)
1534 else:
1535 assert level < lastLevel
1536 while level < lastLevel:
1537 lastLevel -= 1
1538 last_p = last_p.parent()
1539 assert last_p
1540 assert lastLevel >= 0
1541 p = last_p.insertAfter()
1542 last_p = p
1543 lastLevel = level
1544 #@-<< Link a new position p into the outline >>
1545 #@+<< Set the headline string, skipping over the leader >>
1546 #@+node:ekr.20031218072017.3217: *4* << Set the headline string, skipping over the leader >>
1547 j = 0
1548 while g.match(s, j, '\t') or g.match(s, j, ' '):
1549 j += 1
1550 if g.match(s, j, "+ ") or g.match(s, j, "- "):
1551 j += 2
1552 p.initHeadString(s[j:])
1553 #@-<< Set the headline string, skipping over the leader >>
1554 #@+<< Count the number of following body lines >>
1555 #@+node:ekr.20031218072017.3218: *4* << Count the number of following body lines >>
1556 bodyLines = 0
1557 index += 1 # Skip the headline.
1558 while index < len(strings):
1559 s = strings[index]
1560 level, junk = self.headlineLevel(s)
1561 level -= firstLevel
1562 if level >= 0:
1563 break
1564 # Remove first backslash of the body line.
1565 if g.match(s, 0, '\\'):
1566 strings[index] = s[1:]
1567 bodyLines += 1
1568 index += 1
1569 #@-<< Count the number of following body lines >>
1570 #@+<< Add the lines to the body text of p >>
1571 #@+node:ekr.20031218072017.3219: *4* << Add the lines to the body text of p >>
1572 if bodyLines > 0:
1573 body = ""
1574 n = index - bodyLines
1575 while n < index:
1576 body += strings[n].rstrip()
1577 if n != index - 1:
1578 body += "\n"
1579 n += 1
1580 p.setBodyString(body)
1581 #@-<< Add the lines to the body text of p >>
1582 p.setDirty()
1583 else: index += 1
1584 assert progress < index
1585 if theRoot:
1586 theRoot.setDirty()
1587 c.setChanged()
1588 c.redraw()
1589 return theRoot
1590 #@+node:ekr.20031218072017.3222: *3* MORE.headlineLevel
1591 def headlineLevel(self, s):
1592 """return the headline level of s,or -1 if the string is not a MORE headline."""
1593 level = 0
1594 i = 0
1595 while i < len(s) and s[i] in ' \t': # 2016/10/06: allow blanks or tabs.
1596 level += 1
1597 i += 1
1598 plusFlag = g.match(s, i, "+")
1599 if g.match(s, i, "+ ") or g.match(s, i, "- "):
1600 return level, plusFlag
1601 return -1, plusFlag
1602 #@+node:ekr.20031218072017.3223: *3* MORE.check & check_lines
1603 def check(self, s):
1604 s = s.replace("\r", "")
1605 strings = g.splitLines(s)
1606 return self.check_lines(strings)
1608 def check_lines(self, strings):
1610 if not strings:
1611 return False
1612 level1, plusFlag = self.headlineLevel(strings[0])
1613 if level1 == -1:
1614 return False
1615 # Check the level of all headlines.
1616 lastLevel = level1
1617 for s in strings:
1618 level, newFlag = self.headlineLevel(s)
1619 if level == -1:
1620 return True # A body line.
1621 if level < level1 or level > lastLevel + 1:
1622 return False # improper level.
1623 if level > lastLevel and not plusFlag:
1624 return False # parent of this node has no children.
1625 if level == lastLevel and plusFlag:
1626 return False # last node has missing child.
1627 lastLevel = level
1628 plusFlag = newFlag
1629 return True
1630 #@-others
1631#@+node:ekr.20130823083943.12596: ** class RecursiveImportController
1632class RecursiveImportController:
1633 """Recursively import all python files in a directory and clean the result."""
1634 #@+others
1635 #@+node:ekr.20130823083943.12615: *3* ric.ctor
1636 def __init__(self, c, kind,
1637 add_context=None, # Override setting only if True/False
1638 add_file_context=None, # Override setting only if True/False
1639 add_path=True,
1640 recursive=True,
1641 safe_at_file=True,
1642 theTypes=None,
1643 ignore_pattern=None,
1644 verbose=True, # legacy value.
1645 ):
1646 """Ctor for RecursiveImportController class."""
1647 self.c = c
1648 self.add_path = add_path
1649 self.file_pattern = re.compile(r'^(@@|@)(auto|clean|edit|file|nosent)')
1650 self.ignore_pattern = ignore_pattern or re.compile(r'\.git|node_modules')
1651 self.kind = kind # in ('@auto', '@clean', '@edit', '@file', '@nosent')
1652 self.recursive = recursive
1653 self.root = None
1654 self.safe_at_file = safe_at_file
1655 self.theTypes = theTypes
1656 self.verbose = verbose
1657 # #1605:
1659 def set_bool(setting, val):
1660 if val not in (True, False):
1661 return
1662 c.config.set(None, 'bool', setting, val, warn=True)
1664 set_bool('add-context-to-headlines', add_context)
1665 set_bool('add-file-context-to-headlines', add_file_context)
1666 #@+node:ekr.20130823083943.12613: *3* ric.run & helpers
1667 def run(self, dir_):
1668 """
1669 Import all files whose extension matches self.theTypes in dir_.
1670 In fact, dir_ can be a path to a single file.
1671 """
1672 if self.kind not in ('@auto', '@clean', '@edit', '@file', '@nosent'):
1673 g.es('bad kind param', self.kind, color='red')
1674 try:
1675 c = self.c
1676 p1 = self.root = c.p
1677 t1 = time.time()
1678 g.app.disable_redraw = True
1679 bunch = c.undoer.beforeChangeTree(p1)
1680 # Leo 5.6: Always create a new last top-level node.
1681 last = c.lastTopLevel()
1682 parent = last.insertAfter()
1683 parent.v.h = 'imported files'
1684 # Leo 5.6: Special case for a single file.
1685 self.n_files = 0
1686 if g.os_path_isfile(dir_):
1687 if self.verbose:
1688 g.es_print('\nimporting file:', dir_)
1689 self.import_one_file(dir_, parent)
1690 else:
1691 self.import_dir(dir_, parent)
1692 self.post_process(parent, dir_) # Fix # 1033.
1693 c.undoer.afterChangeTree(p1, 'recursive-import', bunch)
1694 except Exception:
1695 g.es_print('Exception in recursive import')
1696 g.es_exception()
1697 finally:
1698 g.app.disable_redraw = False
1699 for p2 in parent.self_and_subtree(copy=False):
1700 p2.contract()
1701 c.redraw(parent)
1702 t2 = time.time()
1703 n = len(list(parent.self_and_subtree()))
1704 g.es_print(
1705 f"imported {n} node{g.plural(n)} "
1706 f"in {self.n_files} file{g.plural(self.n_files)} "
1707 f"in {t2 - t1:2.2f} seconds")
1708 #@+node:ekr.20130823083943.12597: *4* ric.import_dir
1709 def import_dir(self, dir_, parent):
1710 """Import selected files from dir_, a directory."""
1711 if g.os_path_isfile(dir_):
1712 files = [dir_]
1713 else:
1714 if self.verbose:
1715 g.es_print('importing directory:', dir_)
1716 files = os.listdir(dir_)
1717 dirs, files2 = [], []
1718 for path in files:
1719 try:
1720 # Fix #408. Catch path exceptions.
1721 # The idea here is to keep going on small errors.
1722 path = g.os_path_join(dir_, path)
1723 if g.os_path_isfile(path):
1724 name, ext = g.os_path_splitext(path)
1725 if ext in self.theTypes:
1726 files2.append(path)
1727 elif self.recursive:
1728 if not self.ignore_pattern.search(path):
1729 dirs.append(path)
1730 except OSError:
1731 g.es_print('Exception computing', path)
1732 g.es_exception()
1733 if files or dirs:
1734 assert parent and parent.v != self.root.v, g.callers()
1735 parent = parent.insertAsLastChild()
1736 parent.v.h = dir_
1737 if files2:
1738 for f in files2:
1739 if not self.ignore_pattern.search(f):
1740 self.import_one_file(f, parent=parent)
1741 if dirs:
1742 assert self.recursive
1743 for dir_ in sorted(dirs):
1744 self.import_dir(dir_, parent)
1745 #@+node:ekr.20170404103953.1: *4* ric.import_one_file
1746 def import_one_file(self, path, parent):
1747 """Import one file to the last top-level node."""
1748 c = self.c
1749 self.n_files += 1
1750 assert parent and parent.v != self.root.v, g.callers()
1751 if self.kind == '@edit':
1752 p = parent.insertAsLastChild()
1753 p.v.h = '@edit ' + path.replace('\\', '/') # 2021/02/19: bug fix: add @edit.
1754 s, e = g.readFileIntoString(path, kind=self.kind)
1755 p.v.b = s
1756 return
1757 # #1484: Use this for @auto as well.
1758 c.importCommands.importFilesCommand(
1759 files=[path],
1760 parent=parent,
1761 shortFn=True,
1762 treeType='@file', # '@auto','@clean','@nosent' cause problems.
1763 verbose=self.verbose, # Leo 6.6.
1764 )
1765 p = parent.lastChild()
1766 p.h = self.kind + p.h[5:] # Honor the requested kind.
1767 if self.safe_at_file:
1768 p.v.h = '@' + p.v.h
1769 #@+node:ekr.20130823083943.12607: *4* ric.post_process & helpers
1770 def post_process(self, p, prefix):
1771 """
1772 Traverse p's tree, replacing all nodes that start with prefix
1773 by the smallest equivalent @path or @file node.
1774 """
1775 self.fix_back_slashes(p)
1776 prefix = prefix.replace('\\', '/')
1777 if self.kind not in ('@auto', '@edit'):
1778 self.remove_empty_nodes(p)
1779 if p.firstChild():
1780 self.minimize_headlines(p.firstChild(), prefix)
1781 self.clear_dirty_bits(p)
1782 self.add_class_names(p)
1783 #@+node:ekr.20180524100258.1: *5* ric.add_class_names
1784 def add_class_names(self, p):
1785 """Add class names to headlines for all descendant nodes."""
1786 # pylint: disable=no-else-continue
1787 after, class_name = None, None
1788 class_paren_pattern = re.compile(r'(.*)\(.*\)\.(.*)')
1789 paren_pattern = re.compile(r'(.*)\(.*\.py\)')
1790 for p in p.self_and_subtree(copy=False):
1791 # Part 1: update the status.
1792 m = self.file_pattern.match(p.h)
1793 if m:
1794 # prefix = m.group(1)
1795 # fn = g.shortFileName(p.h[len(prefix):].strip())
1796 after, class_name = None, None
1797 continue
1798 elif p.h.startswith('@path '):
1799 after, class_name = None, None
1800 elif p.h.startswith('class '):
1801 class_name = p.h[5:].strip()
1802 if class_name:
1803 after = p.nodeAfterTree()
1804 continue
1805 elif p == after:
1806 after, class_name = None, None
1807 # Part 2: update the headline.
1808 if class_name:
1809 if p.h.startswith(class_name):
1810 m = class_paren_pattern.match(p.h)
1811 if m:
1812 p.h = f"{m.group(1)}.{m.group(2)}".rstrip()
1813 else:
1814 p.h = f"{class_name}.{p.h}"
1815 else:
1816 m = paren_pattern.match(p.h)
1817 if m:
1818 p.h = m.group(1).rstrip()
1819 # elif fn:
1820 # tag = ' (%s)' % fn
1821 # if not p.h.endswith(tag):
1822 # p.h += tag
1823 #@+node:ekr.20130823083943.12608: *5* ric.clear_dirty_bits
1824 def clear_dirty_bits(self, p):
1825 c = self.c
1826 c.clearChanged() # Clears *all* dirty bits.
1827 for p in p.self_and_subtree(copy=False):
1828 p.clearDirty()
1829 #@+node:ekr.20130823083943.12609: *5* ric.dump_headlines
1830 def dump_headlines(self, p):
1831 # show all headlines.
1832 for p in p.self_and_subtree(copy=False):
1833 print(p.h)
1834 #@+node:ekr.20130823083943.12610: *5* ric.fix_back_slashes
1835 def fix_back_slashes(self, p):
1836 """Convert backslash to slash in all headlines."""
1837 for p in p.self_and_subtree(copy=False):
1838 s = p.h.replace('\\', '/')
1839 if s != p.h:
1840 p.v.h = s
1841 #@+node:ekr.20130823083943.12611: *5* ric.minimize_headlines & helper
1842 def minimize_headlines(self, p, prefix):
1843 """Create @path nodes to minimize the paths required in descendant nodes."""
1844 if prefix and not prefix.endswith('/'):
1845 prefix = prefix + '/'
1846 m = self.file_pattern.match(p.h)
1847 if m:
1848 # It's an @file node of some kind. Strip off the prefix.
1849 kind = m.group(0)
1850 path = p.h[len(kind) :].strip()
1851 stripped = self.strip_prefix(path, prefix)
1852 p.h = f"{kind} {stripped or path}"
1853 # Put the *full* @path directive in the body.
1854 if self.add_path and prefix:
1855 tail = g.os_path_dirname(stripped).rstrip('/')
1856 p.b = f"@path {prefix}{tail}\n{p.b}"
1857 else:
1858 # p.h is a path.
1859 path = p.h
1860 stripped = self.strip_prefix(path, prefix)
1861 p.h = f"@path {stripped or path}"
1862 for p in p.children():
1863 self.minimize_headlines(p, prefix + stripped)
1864 #@+node:ekr.20170404134052.1: *6* ric.strip_prefix
1865 def strip_prefix(self, path, prefix):
1866 """Strip the prefix from the path and return the result."""
1867 if path.startswith(prefix):
1868 return path[len(prefix) :]
1869 return '' # A signal.
1870 #@+node:ekr.20130823083943.12612: *5* ric.remove_empty_nodes
1871 def remove_empty_nodes(self, p):
1872 """Remove empty nodes. Not called for @auto or @edit trees."""
1873 c = self.c
1874 aList = [
1875 p2 for p2 in p.self_and_subtree()
1876 if not p2.b and not p2.hasChildren()]
1877 if aList:
1878 c.deletePositionsInList(aList) # Don't redraw.
1879 #@-others
1880#@+node:ekr.20161006071801.1: ** class TabImporter
1881class TabImporter:
1882 """
1883 A class to import a file whose outline levels are indicated by
1884 leading tabs or blanks (but not both).
1885 """
1887 def __init__(self, c, separate=True):
1888 """Ctor for the TabImporter class."""
1889 self.c = c
1890 self.root = None
1891 self.separate = separate
1892 self.stack = []
1893 #@+others
1894 #@+node:ekr.20161006071801.2: *3* tabbed.check
1895 def check(self, lines, warn=True):
1896 """Return False and warn if lines contains mixed leading tabs/blanks."""
1897 blanks, tabs = 0, 0
1898 for s in lines:
1899 lws = self.lws(s)
1900 if '\t' in lws:
1901 tabs += 1
1902 if ' ' in lws:
1903 blanks += 1
1904 if tabs and blanks:
1905 if warn:
1906 g.es_print('intermixed leading blanks and tabs.')
1907 return False
1908 return True
1909 #@+node:ekr.20161006071801.3: *3* tabbed.dump_stack
1910 def dump_stack(self):
1911 """Dump the stack, containing (level, p) tuples."""
1912 g.trace('==========')
1913 for i, data in enumerate(self.stack):
1914 level, p = data
1915 print(f"{i:2} {level} {p.h!r}")
1916 #@+node:ekr.20161006073129.1: *3* tabbed.import_files
1917 def import_files(self, files):
1918 """Import a list of tab-delimited files."""
1919 c, u = self.c, self.c.undoer
1920 if files:
1921 p = None
1922 for fn in files:
1923 try:
1924 g.setGlobalOpenDir(fn)
1925 s = open(fn).read()
1926 s = s.replace('\r', '')
1927 except Exception:
1928 continue
1929 if s.strip() and self.check(g.splitLines(s)):
1930 undoData = u.beforeInsertNode(c.p)
1931 last = c.lastTopLevel()
1932 self.root = p = last.insertAfter()
1933 self.scan(s)
1934 p.h = g.shortFileName(fn)
1935 p.contract()
1936 p.setDirty()
1937 u.afterInsertNode(p, 'Import Tabbed File', undoData)
1938 if p:
1939 c.setChanged()
1940 c.redraw(p)
1941 #@+node:ekr.20161006071801.4: *3* tabbed.lws
1942 def lws(self, s):
1943 """Return the length of the leading whitespace of s."""
1944 for i, ch in enumerate(s):
1945 if ch not in ' \t':
1946 return s[:i]
1947 return s
1948 #@+node:ekr.20161006072958.1: *3* tabbed.prompt_for_files
1949 def prompt_for_files(self):
1950 """Prompt for a list of FreeMind (.mm.html) files and import them."""
1951 c = self.c
1952 types = [
1953 ("All files", "*"),
1954 ]
1955 names = g.app.gui.runOpenFileDialog(c,
1956 title="Import Tabbed File",
1957 filetypes=types,
1958 defaultextension=".html",
1959 multiple=True)
1960 c.bringToFront()
1961 if names:
1962 g.chdir(names[0])
1963 self.import_files(names)
1964 #@+node:ekr.20161006071801.5: *3* tabbed.scan
1965 def scan(self, s1, fn=None, root=None):
1966 """Create the outline corresponding to s1."""
1967 c = self.c
1968 # Self.root can be None if we are called from a script or unit test.
1969 if not self.root:
1970 last = root if root else c.lastTopLevel() # For unit testing.
1971 self.root = last.insertAfter()
1972 if fn:
1973 self.root.h = fn
1974 lines = g.splitLines(s1)
1975 self.stack = []
1976 # Redo the checks in case we are called from a script.
1977 if s1.strip() and self.check(lines):
1978 for s in lines:
1979 if s.strip() or not self.separate:
1980 self.scan_helper(s)
1981 return self.root
1982 #@+node:ekr.20161006071801.6: *3* tabbed.scan_helper
1983 def scan_helper(self, s):
1984 """Update the stack as necessary and return level."""
1985 root, separate, stack = self.root, self.separate, self.stack
1986 if stack:
1987 level, parent = stack[-1]
1988 else:
1989 level, parent = 0, None
1990 lws = len(self.lws(s))
1991 h = s.strip()
1992 if lws == level:
1993 if separate or not parent:
1994 # Replace the top of the stack with a new entry.
1995 if stack:
1996 stack.pop()
1997 grand_parent = stack[-1][1] if stack else root
1998 parent = grand_parent.insertAsLastChild() # lws == level
1999 parent.h = h
2000 stack.append((level, parent),)
2001 elif not parent.h:
2002 parent.h = h
2003 elif lws > level:
2004 # Create a new parent.
2005 level = lws
2006 parent = parent.insertAsLastChild()
2007 parent.h = h
2008 stack.append((level, parent),)
2009 else:
2010 # Find the previous parent.
2011 while stack:
2012 level2, parent2 = stack.pop()
2013 if level2 == lws:
2014 grand_parent = stack[-1][1] if stack else root
2015 parent = grand_parent.insertAsLastChild() # lws < level
2016 parent.h = h
2017 level = lws
2018 stack.append((level, parent),)
2019 break
2020 else:
2021 level = 0
2022 parent = root.insertAsLastChild()
2023 parent.h = h
2024 stack = [(0, parent),]
2025 assert parent and parent == stack[-1][1] # An important invariant.
2026 assert level == stack[-1][0], (level, stack[-1][0])
2027 if not separate:
2028 parent.b = parent.b + self.undent(level, s)
2029 return level
2030 #@+node:ekr.20161006071801.7: *3* tabbed.undent
2031 def undent(self, level, s):
2032 """Unindent all lines of p.b by level."""
2033 if level <= 0:
2034 return s
2035 if s.strip():
2036 lines = g.splitLines(s)
2037 ch = lines[0][0]
2038 assert ch in ' \t', repr(ch)
2039 # Check that all lines start with the proper lws.
2040 lws = ch * level
2041 for s in lines:
2042 if not s.startswith(lws):
2043 g.trace(f"bad indentation: {s!r}")
2044 return s
2045 return ''.join([z[len(lws) :] for z in lines])
2046 return ''
2047 #@-others
2048#@+node:ekr.20200310060123.1: ** class ToDoImporter
2049class ToDoImporter:
2051 def __init__(self, c):
2052 self.c = c
2054 #@+others
2055 #@+node:ekr.20200310103606.1: *3* todo_i.get_tasks_from_file
2056 def get_tasks_from_file(self, path):
2057 """Return the tasks from the given path."""
2058 tag = 'import-todo-text-files'
2059 if not os.path.exists(path):
2060 print(f"{tag}: file not found: {path}")
2061 return []
2062 try:
2063 with open(path, 'r') as f:
2064 contents = f.read()
2065 tasks = self.parse_file_contents(contents)
2066 return tasks
2067 except Exception:
2068 print(f"unexpected exception in {tag}")
2069 g.es_exception()
2070 return []
2071 #@+node:ekr.20200310101028.1: *3* todo_i.import_files
2072 def import_files(self, files):
2073 """
2074 Import all todo.txt files in the given list of file names.
2076 Return a dict: keys are full paths, values are lists of ToDoTasks"
2077 """
2078 d, tag = {}, 'import-todo-text-files'
2079 for path in files:
2080 try:
2081 with open(path, 'r') as f:
2082 contents = f.read()
2083 tasks = self.parse_file_contents(contents)
2084 d[path] = tasks
2085 except Exception:
2086 print(f"unexpected exception in {tag}")
2087 g.es_exception()
2088 return d
2089 #@+node:ekr.20200310062758.1: *3* todo_i.parse_file_contents
2090 # Patterns...
2091 mark_s = r'([x]\ )'
2092 priority_s = r'(\([A-Z]\)\ )'
2093 date_s = r'([0-9]{4}-[0-9]{2}-[0-9]{2}\ )'
2094 task_s = r'\s*(.+)'
2095 line_s = fr"^{mark_s}?{priority_s}?{date_s}?{date_s}?{task_s}$"
2096 line_pat = re.compile(line_s)
2098 def parse_file_contents(self, s):
2099 """
2100 Parse the contents of a file.
2101 Return a list of ToDoTask objects.
2102 """
2103 trace = False
2104 tasks = []
2105 for line in g.splitLines(s):
2106 if not line.strip():
2107 continue
2108 if trace:
2109 print(f"task: {line.rstrip()!s}")
2110 m = self.line_pat.match(line)
2111 if not m:
2112 print(f"invalid task: {line.rstrip()!s}")
2113 continue
2114 # Groups 1, 2 and 5 are context independent.
2115 completed = m.group(1)
2116 priority = m.group(2)
2117 task_s = m.group(5)
2118 if not task_s:
2119 print(f"invalid task: {line.rstrip()!s}")
2120 continue
2121 # Groups 3 and 4 are context dependent.
2122 if m.group(3) and m.group(4):
2123 complete_date = m.group(3)
2124 start_date = m.group(4)
2125 elif completed:
2126 complete_date = m.group(3)
2127 start_date = ''
2128 else:
2129 start_date = m.group(3) or ''
2130 complete_date = ''
2131 if completed and not complete_date:
2132 print(f"no completion date: {line.rstrip()!s}")
2133 tasks.append(ToDoTask(
2134 bool(completed), priority, start_date, complete_date, task_s))
2135 return tasks
2136 #@+node:ekr.20200310100919.1: *3* todo_i.prompt_for_files
2137 def prompt_for_files(self):
2138 """
2139 Prompt for a list of todo.text files and import them.
2141 Return a python dict. Keys are full paths; values are lists of ToDoTask objects.
2142 """
2143 c = self.c
2144 types = [
2145 ("Text files", "*.txt"),
2146 ("All files", "*"),
2147 ]
2148 names = g.app.gui.runOpenFileDialog(c,
2149 title="Import todo.txt File",
2150 filetypes=types,
2151 defaultextension=".txt",
2152 multiple=True,
2153 )
2154 c.bringToFront()
2155 if not names:
2156 return {}
2157 g.chdir(names[0])
2158 d = self.import_files(names)
2159 for key in sorted(d):
2160 tasks = d.get(key)
2161 print(f"tasks in {g.shortFileName(key)}...\n")
2162 for task in tasks:
2163 print(f" {task}")
2164 return d
2165 #@-others
2166#@+node:ekr.20200310063208.1: ** class ToDoTask
2167class ToDoTask:
2168 """A class representing the components of a task line."""
2170 def __init__(self, completed, priority, start_date, complete_date, task_s):
2171 self.completed = completed
2172 self.priority = priority and priority[1] or ''
2173 self.start_date = start_date and start_date.rstrip() or ''
2174 self.complete_date = complete_date and complete_date.rstrip() or ''
2175 self.task_s = task_s.strip()
2176 # Parse tags into separate dictionaries.
2177 self.projects = []
2178 self.contexts = []
2179 self.key_vals = []
2180 self.parse_task()
2182 #@+others
2183 #@+node:ekr.20200310075514.1: *3* task.__repr__ & __str__
2184 def __repr__(self):
2185 start_s = self.start_date if self.start_date else ''
2186 end_s = self.complete_date if self.complete_date else ''
2187 mark_s = '[X]' if self.completed else '[ ]'
2188 result = [
2189 f"Task: "
2190 f"{mark_s} "
2191 f"{self.priority:1} "
2192 f"start: {start_s:10} "
2193 f"end: {end_s:10} "
2194 f"{self.task_s}"
2195 ]
2196 for ivar in ('contexts', 'projects', 'key_vals'):
2197 aList = getattr(self, ivar, None)
2198 if aList:
2199 result.append(f"{' '*13}{ivar}: {aList}")
2200 return '\n'.join(result)
2202 __str__ = __repr__
2203 #@+node:ekr.20200310063138.1: *3* task.parse_task
2204 # Patterns...
2205 project_pat = re.compile(r'(\+\S+)')
2206 context_pat = re.compile(r'(@\S+)')
2207 key_val_pat = re.compile(r'((\S+):(\S+))') # Might be a false match.
2209 def parse_task(self):
2211 trace = False and not g.unitTesting
2212 s = self.task_s
2213 table = (
2214 ('context', self.context_pat, self.contexts),
2215 ('project', self.project_pat, self.projects),
2216 ('key:val', self.key_val_pat, self.key_vals),
2217 )
2218 for kind, pat, aList in table:
2219 for m in re.finditer(pat, s):
2220 pat_s = repr(pat).replace("re.compile('", "").replace("')", "")
2221 pat_s = pat_s.replace(r'\\', '\\')
2222 # Check for false key:val match:
2223 if pat == self.key_val_pat:
2224 key, value = m.group(2), m.group(3)
2225 if ':' in key or ':' in value:
2226 break
2227 tag = m.group(1)
2228 # Add the tag.
2229 if tag in aList:
2230 if trace:
2231 g.trace('Duplicate tag:', tag)
2232 else:
2233 if trace:
2234 g.trace(f"Add {kind} tag: {tag!s}")
2235 aList.append(tag)
2236 # Remove the tag from the task.
2237 s = re.sub(pat, "", s)
2238 if s != self.task_s:
2239 self.task_s = s.strip()
2240 #@-others
2241#@+node:ekr.20141210051628.26: ** class ZimImportController
2242class ZimImportController:
2243 """
2244 A class to import Zim folders and files: http://zim-wiki.org/
2245 First use Zim to export your project to rst files.
2247 Original script by Davy Cottet.
2249 User options:
2250 @int rst_level = 0
2251 @string rst_type
2252 @string zim_node_name
2253 @string path_to_zim
2255 """
2256 #@+others
2257 #@+node:ekr.20141210051628.31: *3* zic.__init__ & zic.reloadSettings
2258 def __init__(self, c):
2259 """Ctor for ZimImportController class."""
2260 self.c = c
2261 self.pathToZim = c.config.getString('path-to-zim')
2262 self.rstLevel = c.config.getInt('zim-rst-level') or 0
2263 self.rstType = c.config.getString('zim-rst-type') or 'rst'
2264 self.zimNodeName = c.config.getString('zim-node-name') or 'Imported Zim Tree'
2265 #@+node:ekr.20141210051628.28: *3* zic.parseZimIndex
2266 def parseZimIndex(self):
2267 """
2268 Parse Zim wiki index.rst and return a list of tuples (level, name, path) or None.
2269 """
2270 # c = self.c
2271 pathToZim = g.os_path_abspath(self.pathToZim)
2272 pathToIndex = g.os_path_join(pathToZim, 'index.rst')
2273 if not g.os_path_exists(pathToIndex):
2274 g.es(f"not found: {pathToIndex}", color='red')
2275 return None
2276 index = open(pathToIndex).read()
2277 parse = re.findall(r'(\t*)-\s`(.+)\s<(.+)>`_', index)
2278 if not parse:
2279 g.es(f"invalid index: {pathToIndex}", color='red')
2280 return None
2281 results = []
2282 for result in parse:
2283 level = len(result[0])
2284 name = result[1].decode('utf-8')
2285 unquote = urllib.parse.unquote
2286 # mypy: error: "str" has no attribute "decode"; maybe "encode"? [attr-defined]
2287 path = [g.os_path_abspath(g.os_path_join(
2288 pathToZim, unquote(result[2]).decode('utf-8')))] # type:ignore
2289 results.append((level, name, path))
2290 return results
2291 #@+node:ekr.20141210051628.29: *3* zic.rstToLastChild
2292 def rstToLastChild(self, p, name, rst):
2293 """Import an rst file as a last child of pos node with the specified name"""
2294 c = self.c
2295 c.importCommands.importFilesCommand(
2296 files=rst,
2297 parent=p,
2298 treeType='@rst',
2299 )
2300 rstNode = p.getLastChild()
2301 rstNode.h = name
2302 return rstNode
2303 #@+node:davy.20141212140940.1: *3* zic.clean
2304 def clean(self, zimNode, rstType):
2305 """Clean useless nodes"""
2306 warning = 'Warning: this node is ignored when writing this file'
2307 for p in zimNode.subtree_iter():
2308 # looking for useless bodies
2309 if p.hasFirstChild() and warning in p.b:
2310 child = p.getFirstChild()
2311 fmt = "@rst-no-head %s declarations"
2312 table = (
2313 fmt % p.h.replace(' ', '_'),
2314 fmt % p.h.replace(rstType, '').strip().replace(' ', '_'),
2315 )
2316 # Replace content with @rest-no-head first child (without title head) and delete it
2317 if child.h in table:
2318 p.b = '\n'.join(child.b.split('\n')[3:])
2319 child.doDelete()
2320 # Replace content of empty body parent node with first child with same name
2321 elif p.h == child.h or (f"{rstType} {child.h}" == p.h):
2322 if not child.hasFirstChild():
2323 p.b = child.b
2324 child.doDelete()
2325 elif not child.hasNext():
2326 p.b = child.b
2327 child.copyTreeFromSelfTo(p)
2328 child.doDelete()
2329 else:
2330 child.h = 'Introduction'
2331 elif p.hasFirstChild(
2332 ) and p.h.startswith("@rst-no-head") and not p.b.strip():
2333 child = p.getFirstChild()
2334 p_no_head = p.h.replace("@rst-no-head", "").strip()
2335 # Replace empty @rst-no-head by its same named chidren
2336 if child.h.strip() == p_no_head and not child.hasFirstChild():
2337 p.h = p_no_head
2338 p.b = child.b
2339 child.doDelete()
2340 elif p.h.startswith("@rst-no-head"):
2341 lines = p.b.split('\n')
2342 p.h = lines[1]
2343 p.b = '\n'.join(lines[3:])
2344 #@+node:ekr.20141210051628.30: *3* zic.run
2345 def run(self):
2346 """Create the zim node as the last top-level node."""
2347 c = self.c
2348 # Make sure a path is given.
2349 if not self.pathToZim:
2350 g.es('Missing setting: @string path_to_zim', color='red')
2351 return
2352 root = c.rootPosition()
2353 while root.hasNext():
2354 root.moveToNext()
2355 zimNode = root.insertAfter()
2356 zimNode.h = self.zimNodeName
2357 # Parse the index file
2358 files = self.parseZimIndex()
2359 if files:
2360 # Do the import
2361 rstNodes = {'0': zimNode,}
2362 for level, name, rst in files:
2363 if level == self.rstLevel:
2364 name = f"{self.rstType} {name}"
2365 rstNodes[
2366 str(
2367 level + 1)] = self.rstToLastChild(rstNodes[str(level)], name, rst)
2368 # Clean nodes
2369 g.es('Start cleaning process. Please wait...', color='blue')
2370 self.clean(zimNode, self.rstType)
2371 g.es('Done', color='blue')
2372 # Select zimNode
2373 c.selectPosition(zimNode)
2374 c.redraw()
2375 #@-others
2376#@+node:ekr.20200424152850.1: ** class LegacyExternalFileImporter
2377class LegacyExternalFileImporter:
2378 """
2379 A class to import external files written by versions of Leo earlier
2380 than 5.0.
2381 """
2382 # Sentinels to ignore, without the leading comment delim.
2383 ignore = ('@+at', '@-at', '@+leo', '@-leo', '@nonl', '@nl', '@-others')
2385 def __init__(self, c):
2386 self.c = c
2388 #@+others
2389 #@+node:ekr.20200424093946.1: *3* class Node
2390 class Node:
2392 def __init__(self, h, level):
2393 """Hold node data."""
2394 self.h = h.strip()
2395 self.level = level
2396 self.lines = []
2397 #@+node:ekr.20200424092652.1: *3* legacy.add
2398 def add(self, line, stack):
2399 """Add a line to the present node."""
2400 if stack:
2401 node = stack[-1]
2402 node.lines.append(line)
2403 else:
2404 print('orphan line: ', repr(line))
2405 #@+node:ekr.20200424160847.1: *3* legacy.compute_delim1
2406 def compute_delim1(self, path):
2407 """Return the opening comment delim for the given file."""
2408 junk, ext = os.path.splitext(path)
2409 if not ext:
2410 return None
2411 language = g.app.extension_dict.get(ext[1:])
2412 if not language:
2413 return None
2414 delim1, delim2, delim3 = g.set_delims_from_language(language)
2415 g.trace(language, delim1 or delim2)
2416 return delim1 or delim2
2417 #@+node:ekr.20200424153139.1: *3* legacy.import_file
2418 def import_file(self, path):
2419 """Import one legacy external file."""
2420 c = self.c
2421 root_h = g.shortFileName(path)
2422 delim1 = self.compute_delim1(path)
2423 if not delim1:
2424 g.es_print('unknown file extension:', color='red')
2425 g.es_print(path)
2426 return
2427 # Read the file into s.
2428 with open(path, 'r') as f:
2429 s = f.read()
2430 # Do nothing if the file is a newer external file.
2431 if delim1 + '@+leo-ver=4' not in s:
2432 g.es_print('not a legacy external file:', color='red')
2433 g.es_print(path)
2434 return
2435 # Compute the local ignore list for this file.
2436 ignore = tuple(delim1 + z for z in self.ignore)
2437 # Handle each line of the file.
2438 nodes: List[Any] = [] # An list of Nodes, in file order.
2439 stack: List[Any] = [] # A stack of Nodes.
2440 for line in g.splitLines(s):
2441 s = line.lstrip()
2442 lws = line[: len(line) - len(line.lstrip())]
2443 if s.startswith(delim1 + '@@'):
2444 self.add(lws + s[2:], stack)
2445 elif s.startswith(ignore):
2446 # Ignore these. Use comments instead of @doc bodies.
2447 pass
2448 elif (
2449 s.startswith(delim1 + '@+others') or
2450 s.startswith(delim1 + '@' + lws + '@+others')
2451 ):
2452 self.add(lws + '@others\n', stack)
2453 elif s.startswith(delim1 + '@<<'):
2454 n = len(delim1 + '@<<')
2455 self.add(lws + '<<' + s[n:].rstrip() + '\n', stack)
2456 elif s.startswith(delim1 + '@+node:'):
2457 # Compute the headline.
2458 if stack:
2459 h = s[8:]
2460 i = h.find(':')
2461 h = h[i + 1 :] if ':' in h else h
2462 else:
2463 h = root_h
2464 # Create a node and push it.
2465 node = self.Node(h, len(stack))
2466 nodes.append(node)
2467 stack.append(node)
2468 elif s.startswith(delim1 + '@-node'):
2469 # End the node.
2470 stack.pop()
2471 elif s.startswith(delim1 + '@'):
2472 print('oops:', repr(s))
2473 else:
2474 self.add(line, stack)
2475 if stack:
2476 print('Unbalanced node sentinels')
2477 # Generate nodes.
2478 last = c.lastTopLevel()
2479 root = last.insertAfter()
2480 root.h = f"imported file: {root_h}"
2481 stack = [root]
2482 for node in nodes:
2483 b = textwrap.dedent(''.join(node.lines))
2484 level = node.level
2485 if level == 0:
2486 root.h = root_h
2487 root.b = b
2488 else:
2489 parent = stack[level - 1]
2490 p = parent.insertAsLastChild()
2491 p.b = b
2492 p.h = node.h
2493 # Good for debugging.
2494 # p.h = f"{level} {node.h}"
2495 stack = stack[:level] + [p]
2496 c.selectPosition(root)
2497 root.expand() # c.expandAllSubheads()
2498 c.redraw()
2499 #@+node:ekr.20200424154553.1: *3* legacy.import_files
2500 def import_files(self, paths):
2501 """Import zero or more files."""
2502 for path in paths:
2503 if os.path.exists(path):
2504 self.import_file(path)
2505 else:
2506 g.es_print(f"not found: {path!r}")
2507 #@+node:ekr.20200424154416.1: *3* legacy.prompt_for_files
2508 def prompt_for_files(self):
2509 """Prompt for a list of legacy external .py files and import them."""
2510 c = self.c
2511 types = [
2512 ("Legacy external files", "*.py"),
2513 ("All files", "*"),
2514 ]
2515 paths = g.app.gui.runOpenFileDialog(c,
2516 title="Import Legacy External Files",
2517 filetypes=types,
2518 defaultextension=".py",
2519 multiple=True)
2520 c.bringToFront()
2521 if paths:
2522 g.chdir(paths[0])
2523 self.import_files(paths)
2524 #@-others
2525#@+node:ekr.20101103093942.5938: ** Commands (leoImport)
2526#@+node:ekr.20160504050255.1: *3* @g.command(import-free-mind-files)
2527@g.command('import-free-mind-files')
2528def import_free_mind_files(event):
2529 """Prompt for free-mind files and import them."""
2530 c = event.get('c')
2531 if c:
2532 FreeMindImporter(c).prompt_for_files()
2534#@+node:ekr.20200424154303.1: *3* @g.command(import-legacy-external-file)
2535@g.command('import-legacy-external-files')
2536def import_legacy_external_files(event):
2537 """Prompt for legacy external files and import them."""
2538 c = event.get('c')
2539 if c:
2540 LegacyExternalFileImporter(c).prompt_for_files()
2541#@+node:ekr.20160504050325.1: *3* @g.command(import-mind-map-files
2542@g.command('import-mind-jet-files')
2543def import_mind_jet_files(event):
2544 """Prompt for mind-jet files and import them."""
2545 c = event.get('c')
2546 if c:
2547 MindMapImporter(c).prompt_for_files()
2548#@+node:ekr.20161006100854.1: *3* @g.command(import-MORE-files)
2549@g.command('import-MORE-files')
2550def import_MORE_files_command(event):
2551 """Prompt for MORE files and import them."""
2552 c = event.get('c')
2553 if c:
2554 MORE_Importer(c).prompt_for_files()
2555#@+node:ekr.20161006072227.1: *3* @g.command(import-tabbed-files)
2556@g.command('import-tabbed-files')
2557def import_tabbed_files_command(event):
2558 """Prompt for tabbed files and import them."""
2559 c = event.get('c')
2560 if c:
2561 TabImporter(c).prompt_for_files()
2562#@+node:ekr.20200310095703.1: *3* @g.command(import-todo-text-files)
2563@g.command('import-todo-text-files')
2564def import_todo_text_files(event):
2565 """Prompt for free-mind files and import them."""
2566 c = event.get('c')
2567 if c:
2568 ToDoImporter(c).prompt_for_files()
2569#@+node:ekr.20141210051628.33: *3* @g.command(import-zim-folder)
2570@g.command('import-zim-folder')
2571def import_zim_command(event):
2572 """
2573 Import a zim folder, http://zim-wiki.org/, as the last top-level node of the outline.
2575 First use Zim to export your project to rst files.
2577 This command requires the following Leo settings::
2579 @int rst_level = 0
2580 @string rst_type
2581 @string zim_node_name
2582 @string path_to_zim
2583 """
2584 c = event.get('c')
2585 if c:
2586 ZimImportController(c).run()
2587#@+node:ekr.20120429125741.10057: *3* @g.command(parse-body)
2588@g.command('parse-body')
2589def parse_body_command(event):
2590 """The parse-body command."""
2591 c = event.get('c')
2592 if c and c.p:
2593 c.importCommands.parse_body(c.p)
2594#@-others
2595#@@language python
2596#@@tabwidth -4
2597#@@pagewidth 70
2598#@@encoding utf-8
2599#@-leo