Coverage for C:\leo.repo\leo-editor\leo\core\leoImport.py: 13%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20031218072017.3206: * @file leoImport.py
4#@@first
5#@+<< imports >>
6#@+node:ekr.20091224155043.6539: ** << imports >> (leoImport)
7import csv
8import io
9import json
10import os
11import re
12import textwrap
13import time
14from typing import Any, List
15import urllib
16#
17# Third-party imports.
18try:
19 import docutils
20 import docutils.core
21 assert docutils
22 assert docutils.core
23except ImportError:
24 # print('leoImport.py: can not import docutils')
25 docutils = None # type:ignore
26try:
27 import lxml
28except ImportError:
29 lxml = None
30#
31# Leo imports...
32from leo.core import leoGlobals as g
33from leo.core import leoNodes
34#
35# Abbreviation.
36StringIO = io.StringIO
37#@-<< imports >>
38#@+others
39#@+node:ekr.20160503145550.1: ** class FreeMindImporter
40class FreeMindImporter:
41 """Importer class for FreeMind (.mmap) files."""
43 def __init__(self, c):
44 """ctor for FreeMind Importer class."""
45 self.c = c
46 self.count = 0
47 self.d = {}
48 #@+others
49 #@+node:ekr.20170222084048.1: *3* freemind.add_children
50 def add_children(self, parent, element):
51 """
52 parent is the parent position, element is the parent element.
53 Recursively add all the child elements as descendants of parent_p.
54 """
55 p = parent.insertAsLastChild()
56 attrib_text = element.attrib.get('text', '').strip()
57 tag = element.tag if isinstance(element.tag, str) else ''
58 text = element.text or ''
59 if not tag:
60 text = text.strip()
61 p.h = attrib_text or tag or 'Comment'
62 p.b = text if text.strip() else ''
63 for child in element:
64 self.add_children(p, child)
65 #@+node:ekr.20160503125844.1: *3* freemind.create_outline
66 def create_outline(self, path):
67 """Create a tree of nodes from a FreeMind file."""
68 c = self.c
69 junk, fileName = g.os_path_split(path)
70 undoData = c.undoer.beforeInsertNode(c.p)
71 try:
72 self.import_file(path)
73 c.undoer.afterInsertNode(c.p, 'Import', undoData)
74 except Exception:
75 g.es_print('Exception importing FreeMind file', g.shortFileName(path))
76 g.es_exception()
77 return c.p
78 #@+node:ekr.20160503191518.4: *3* freemind.import_file
79 def import_file(self, path):
80 """The main line of the FreeMindImporter class."""
81 c = self.c
82 sfn = g.shortFileName(path)
83 if g.os_path_exists(path):
84 htmltree = lxml.html.parse(path)
85 root = htmltree.getroot()
86 body = root.findall('body')[0]
87 if body is None:
88 g.error(f"no body in: {sfn}")
89 else:
90 root_p = c.lastTopLevel().insertAfter()
91 root_p.h = g.shortFileName(path)
92 for child in body:
93 if child != body:
94 self.add_children(root_p, child)
95 c.selectPosition(root_p)
96 c.redraw()
97 else:
98 g.error(f"file not found: {sfn}")
99 #@+node:ekr.20160503145113.1: *3* freemind.import_files
100 def import_files(self, files):
101 """Import a list of FreeMind (.mmap) files."""
102 c = self.c
103 if files:
104 self.tab_width = c.getTabWidth(c.p)
105 for fileName in files:
106 g.setGlobalOpenDir(fileName)
107 p = self.create_outline(fileName)
108 p.contract()
109 p.setDirty()
110 c.setChanged()
111 c.redraw(p)
112 #@+node:ekr.20160504043823.1: *3* freemind.prompt_for_files
113 def prompt_for_files(self):
114 """Prompt for a list of FreeMind (.mm.html) files and import them."""
115 if not lxml:
116 g.trace("FreeMind importer requires lxml")
117 return
118 c = self.c
119 types = [
120 ("FreeMind files", "*.mm.html"),
121 ("All files", "*"),
122 ]
123 names = g.app.gui.runOpenFileDialog(c,
124 title="Import FreeMind File",
125 filetypes=types,
126 defaultextension=".html",
127 multiple=True)
128 c.bringToFront()
129 if names:
130 g.chdir(names[0])
131 self.import_files(names)
132 #@-others
133#@+node:ekr.20160504144241.1: ** class JSON_Import_Helper
134class JSON_Import_Helper:
135 """
136 A class that helps client scripts import .json files.
138 Client scripts supply data describing how to create Leo outlines from
139 the .json data.
140 """
142 def __init__(self, c):
143 """ctor for the JSON_Import_Helper class."""
144 self.c = c
145 self.vnodes_dict = {}
146 #@+others
147 #@+node:ekr.20160504144353.1: *3* json.create_nodes (generalize)
148 def create_nodes(self, parent, parent_d):
149 """Create the tree of nodes rooted in parent."""
150 d = self.gnx_dict
151 for child_gnx in parent_d.get('children'):
152 d2 = d.get(child_gnx)
153 if child_gnx in self.vnodes_dict:
154 # It's a clone.
155 v = self.vnodes_dict.get(child_gnx)
156 n = parent.numberOfChildren()
157 child = leoNodes.Position(v)
158 child._linkAsNthChild(parent, n)
159 # Don't create children again.
160 else:
161 child = parent.insertAsLastChild()
162 child.h = d2.get('h') or '<**no h**>'
163 child.b = d2.get('b') or ''
164 if d2.get('gnx'):
165 child.v.fileIndex = gnx = d2.get('gnx') # 2021/06/23: found by mypy.
166 self.vnodes_dict[gnx] = child.v
167 if d2.get('ua'):
168 child.u = d2.get('ua')
169 self.create_nodes(child, d2)
170 #@+node:ekr.20160504144241.2: *3* json.create_outline (generalize)
171 def create_outline(self, path):
172 c = self.c
173 junk, fileName = g.os_path_split(path)
174 undoData = c.undoer.beforeInsertNode(c.p)
175 # Create the top-level headline.
176 p = c.lastTopLevel().insertAfter()
177 fn = g.shortFileName(path).strip()
178 if fn.endswith('.json'):
179 fn = fn[:-5]
180 p.h = fn
181 self.scan(path, p)
182 c.undoer.afterInsertNode(p, 'Import', undoData)
183 return p
184 #@+node:ekr.20160504144314.1: *3* json.scan (generalize)
185 def scan(self, s, parent):
186 """Create an outline from a MindMap (.csv) file."""
187 c, d, self.gnx_dict = self.c, json.loads(s), {}
188 for d2 in d.get('nodes', []):
189 gnx = d2.get('gnx')
190 self.gnx_dict[gnx] = d2
191 top_d = d.get('top')
192 if top_d:
193 # Don't set parent.h or parent.gnx or parent.v.u.
194 parent.b = top_d.get('b') or ''
195 self.create_nodes(parent, top_d)
196 c.redraw()
197 return bool(top_d)
198 #@-others
199#@+node:ekr.20071127175948: ** class LeoImportCommands
200class LeoImportCommands:
201 """
202 A class implementing all of Leo's import/export code. This class
203 uses **importers** in the leo/plugins/importers folder.
205 For more information, see leo/plugins/importers/howto.txt.
206 """
207 #@+others
208 #@+node:ekr.20031218072017.3207: *3* ic.__init__& ic.reload_settings
209 def __init__(self, c):
210 """ctor for LeoImportCommands class."""
211 self.c = c
212 self.encoding = 'utf-8'
213 self.errors = 0
214 self.fileName = None # The original file name, say x.cpp
215 self.fileType = None # ".py", ".c", etc.
216 self.methodName = None # x, as in < < x methods > > =
217 self.output_newline = g.getOutputNewline(c=c) # Value of @bool output_newline
218 self.tab_width = c.tab_width
219 self.treeType = "@file" # None or "@file"
220 self.verbose = True # Leo 6.6
221 self.webType = "@noweb" # "cweb" or "noweb"
222 self.web_st = [] # noweb symbol table.
223 self.reload_settings()
225 def reload_settings(self):
226 pass
228 reloadSettings = reload_settings
229 #@+node:ekr.20031218072017.3289: *3* ic.Export
230 #@+node:ekr.20031218072017.3290: *4* ic.convertCodePartToWeb & helpers
231 def convertCodePartToWeb(self, s, i, p, result):
232 """
233 # Headlines not containing a section reference are ignored in noweb
234 and generate index index in cweb.
235 """
236 ic = self
237 nl = ic.output_newline
238 head_ref = ic.getHeadRef(p)
239 file_name = ic.getFileName(p)
240 if g.match_word(s, i, "@root"):
241 i = g.skip_line(s, i)
242 ic.appendRefToFileName(file_name, result)
243 elif g.match_word(s, i, "@c") or g.match_word(s, i, "@code"):
244 i = g.skip_line(s, i)
245 ic.appendHeadRef(p, file_name, head_ref, result)
246 elif g.match_word(p.h, 0, "@file"):
247 # Only do this if nothing else matches.
248 ic.appendRefToFileName(file_name, result)
249 i = g.skip_line(s, i) # 4/28/02
250 else:
251 ic.appendHeadRef(p, file_name, head_ref, result)
252 i, result = ic.copyPart(s, i, result)
253 return i, result.strip() + nl
254 #@+at %defs a b c
255 #@+node:ekr.20140630085837.16720: *5* ic.appendHeadRef
256 def appendHeadRef(self, p, file_name, head_ref, result):
257 ic = self
258 nl = ic.output_newline
259 if ic.webType == "cweb":
260 if head_ref:
261 escaped_head_ref = head_ref.replace("@", "@@")
262 result += "@<" + escaped_head_ref + "@>=" + nl
263 else:
264 result += "@^" + p.h.strip() + "@>" + nl
265 # Convert the headline to an index entry.
266 result += "@c" + nl
267 # @c denotes a new section.
268 else:
269 if head_ref:
270 pass
271 elif p == ic.c.p:
272 head_ref = file_name or "*"
273 else:
274 head_ref = "@others"
275 # 2019/09/12
276 result += (g.angleBrackets(head_ref) + "=" + nl)
277 #@+node:ekr.20140630085837.16719: *5* ic.appendRefToFileName
278 def appendRefToFileName(self, file_name, result):
279 ic = self
280 nl = ic.output_newline
281 if ic.webType == "cweb":
282 if not file_name:
283 result += "@<root@>=" + nl
284 else:
285 result += "@(" + file_name + "@>" + nl
286 # @(...@> denotes a file.
287 else:
288 if not file_name:
289 file_name = "*"
290 # 2019/09/12.
291 lt = "<<"
292 rt = ">>"
293 result += (lt + file_name + rt + "=" + nl)
294 #@+node:ekr.20140630085837.16721: *5* ic.getHeadRef
295 def getHeadRef(self, p):
296 """
297 Look for either noweb or cweb brackets.
298 Return everything between those brackets.
299 """
300 h = p.h.strip()
301 if g.match(h, 0, "<<"):
302 i = h.find(">>", 2)
303 elif g.match(h, 0, "<@"):
304 i = h.find("@>", 2)
305 else:
306 return h
307 return h[2:i].strip()
308 #@+node:ekr.20031218072017.3292: *5* ic.getFileName
309 def getFileName(self, p):
310 """Return the file name from an @file or @root node."""
311 h = p.h.strip()
312 if g.match(h, 0, "@file") or g.match(h, 0, "@root"):
313 line = h[5:].strip()
314 # set j & k so line[j:k] is the file name.
315 if g.match(line, 0, "<"):
316 j, k = 1, line.find(">", 1)
317 elif g.match(line, 0, '"'):
318 j, k = 1, line.find('"', 1)
319 else:
320 j, k = 0, line.find(" ", 0)
321 if k == -1:
322 k = len(line)
323 file_name = line[j:k].strip()
324 else:
325 file_name = ''
326 return file_name
327 #@+node:ekr.20031218072017.3296: *4* ic.convertDocPartToWeb (handle @ %def)
328 def convertDocPartToWeb(self, s, i, result):
329 nl = self.output_newline
330 if g.match_word(s, i, "@doc"):
331 i = g.skip_line(s, i)
332 elif g.match(s, i, "@ ") or g.match(s, i, "@\t") or g.match(s, i, "@*"):
333 i += 2
334 elif g.match(s, i, "@\n"):
335 i += 1
336 i = g.skip_ws_and_nl(s, i)
337 i, result2 = self.copyPart(s, i, "")
338 if result2:
339 # Break lines after periods.
340 result2 = result2.replace(". ", "." + nl)
341 result2 = result2.replace(". ", "." + nl)
342 result += nl + "@" + nl + result2.strip() + nl + nl
343 else:
344 # All nodes should start with '@', even if the doc part is empty.
345 result += nl + "@ " if self.webType == "cweb" else nl + "@" + nl
346 return i, result
347 #@+node:ekr.20031218072017.3297: *4* ic.convertVnodeToWeb
348 def convertVnodeToWeb(self, v):
349 """
350 This code converts a VNode to noweb text as follows:
352 Convert @doc to @
353 Convert @root or @code to < < name > >=, assuming the headline contains < < name > >
354 Ignore other directives
355 Format doc parts so they fit in pagewidth columns.
356 Output code parts as is.
357 """
358 c = self.c
359 if not v or not c:
360 return ""
361 startInCode = not c.config.at_root_bodies_start_in_doc_mode
362 nl = self.output_newline
363 docstart = nl + "@ " if self.webType == "cweb" else nl + "@" + nl
364 s = v.b
365 lb = "@<" if self.webType == "cweb" else "<<"
366 i, result, docSeen = 0, "", False
367 while i < len(s):
368 progress = i
369 i = g.skip_ws_and_nl(s, i)
370 if self.isDocStart(s, i) or g.match_word(s, i, "@doc"):
371 i, result = self.convertDocPartToWeb(s, i, result)
372 docSeen = True
373 elif (
374 g.match_word(s, i, "@code") or
375 g.match_word(s, i, "@root") or
376 g.match_word(s, i, "@c") or
377 g.match(s, i, lb)
378 ):
379 if not docSeen:
380 docSeen = True
381 result += docstart
382 i, result = self.convertCodePartToWeb(s, i, v, result)
383 elif self.treeType == "@file" or startInCode:
384 if not docSeen:
385 docSeen = True
386 result += docstart
387 i, result = self.convertCodePartToWeb(s, i, v, result)
388 else:
389 i, result = self.convertDocPartToWeb(s, i, result)
390 docSeen = True
391 assert progress < i
392 result = result.strip()
393 if result:
394 result += nl
395 return result
396 #@+node:ekr.20031218072017.3299: *4* ic.copyPart
397 # Copies characters to result until the end of the present section is seen.
399 def copyPart(self, s, i, result):
401 lb = "@<" if self.webType == "cweb" else "<<"
402 rb = "@>" if self.webType == "cweb" else ">>"
403 theType = self.webType
404 while i < len(s):
405 progress = j = i # We should be at the start of a line here.
406 i = g.skip_nl(s, i)
407 i = g.skip_ws(s, i)
408 if self.isDocStart(s, i):
409 return i, result
410 if (g.match_word(s, i, "@doc") or
411 g.match_word(s, i, "@c") or
412 g.match_word(s, i, "@root") or
413 g.match_word(s, i, "@code") # 2/25/03
414 ):
415 return i, result
416 # 2019/09/12
417 lt = "<<"
418 rt = ">>="
419 if g.match(s, i, lt) and g.find_on_line(s, i, rt) > -1:
420 return i, result
421 # Copy the entire line, escaping '@' and
422 # Converting @others to < < @ others > >
423 i = g.skip_line(s, j)
424 line = s[j:i]
425 if theType == "cweb":
426 line = line.replace("@", "@@")
427 else:
428 j = g.skip_ws(line, 0)
429 if g.match(line, j, "@others"):
430 line = line.replace("@others", lb + "@others" + rb)
431 elif g.match(line, 0, "@"):
432 # Special case: do not escape @ %defs.
433 k = g.skip_ws(line, 1)
434 if not g.match(line, k, "%defs"):
435 line = "@" + line
436 result += line
437 assert progress < i
438 return i, result.rstrip()
439 #@+node:ekr.20031218072017.1462: *4* ic.exportHeadlines
440 def exportHeadlines(self, fileName):
441 p = self.c.p
442 nl = self.output_newline
443 if not p:
444 return
445 self.setEncoding()
446 firstLevel = p.level()
447 try:
448 with open(fileName, 'w') as theFile:
449 for p in p.self_and_subtree(copy=False):
450 head = p.moreHead(firstLevel, useVerticalBar=True)
451 theFile.write(head + nl)
452 except IOError:
453 g.warning("can not open", fileName)
454 #@+node:ekr.20031218072017.1147: *4* ic.flattenOutline
455 def flattenOutline(self, fileName):
456 """
457 A helper for the flatten-outline command.
459 Export the selected outline to an external file.
460 The outline is represented in MORE format.
461 """
462 c = self.c
463 nl = self.output_newline
464 p = c.p
465 if not p:
466 return
467 self.setEncoding()
468 firstLevel = p.level()
469 try:
470 theFile = open(fileName, 'wb') # Fix crasher: open in 'wb' mode.
471 except IOError:
472 g.warning("can not open", fileName)
473 return
474 for p in p.self_and_subtree(copy=False):
475 s = p.moreHead(firstLevel) + nl
476 s = g.toEncodedString(s, encoding=self.encoding, reportErrors=True)
477 theFile.write(s)
478 s = p.moreBody() + nl # Inserts escapes.
479 if s.strip():
480 s = g.toEncodedString(s, self.encoding, reportErrors=True)
481 theFile.write(s)
482 theFile.close()
483 #@+node:ekr.20031218072017.1148: *4* ic.outlineToWeb
484 def outlineToWeb(self, fileName, webType):
485 c = self.c
486 nl = self.output_newline
487 current = c.p
488 if not current:
489 return
490 self.setEncoding()
491 self.webType = webType
492 try:
493 theFile = open(fileName, 'w')
494 except IOError:
495 g.warning("can not open", fileName)
496 return
497 self.treeType = "@file"
498 # Set self.treeType to @root if p or an ancestor is an @root node.
499 for p in current.parents():
500 flag, junk = g.is_special(p.b, "@root")
501 if flag:
502 self.treeType = "@root"
503 break
504 for p in current.self_and_subtree(copy=False):
505 s = self.convertVnodeToWeb(p)
506 if s:
507 theFile.write(s)
508 if s[-1] != '\n':
509 theFile.write(nl)
510 theFile.close()
511 #@+node:ekr.20031218072017.3300: *4* ic.removeSentinelsCommand
512 def removeSentinelsCommand(self, paths, toString=False):
513 c = self.c
514 self.setEncoding()
515 for fileName in paths:
516 g.setGlobalOpenDir(fileName)
517 path, self.fileName = g.os_path_split(fileName)
518 s, e = g.readFileIntoString(fileName, self.encoding)
519 if s is None:
520 return None
521 if e:
522 self.encoding = e
523 #@+<< set delims from the header line >>
524 #@+node:ekr.20031218072017.3302: *5* << set delims from the header line >>
525 # Skip any non @+leo lines.
526 i = 0
527 while i < len(s) and g.find_on_line(s, i, "@+leo") == -1:
528 i = g.skip_line(s, i)
529 # Get the comment delims from the @+leo sentinel line.
530 at = self.c.atFileCommands
531 j = g.skip_line(s, i)
532 line = s[i:j]
533 valid, junk, start_delim, end_delim, junk = at.parseLeoSentinel(line)
534 if not valid:
535 if not toString:
536 g.es("invalid @+leo sentinel in", fileName)
537 return None
538 if end_delim:
539 line_delim = None
540 else:
541 line_delim, start_delim = start_delim, None
542 #@-<< set delims from the header line >>
543 s = self.removeSentinelLines(s, line_delim, start_delim, end_delim)
544 ext = c.config.remove_sentinels_extension
545 if not ext:
546 ext = ".txt"
547 if ext[0] == '.':
548 newFileName = g.os_path_finalize_join(path, fileName + ext) # 1341
549 else:
550 head, ext2 = g.os_path_splitext(fileName)
551 newFileName = g.os_path_finalize_join(path, head + ext + ext2) # 1341
552 if toString:
553 return s
554 #@+<< Write s into newFileName >>
555 #@+node:ekr.20031218072017.1149: *5* << Write s into newFileName >> (remove-sentinels)
556 # Remove sentinels command.
557 try:
558 with open(newFileName, 'w') as theFile:
559 theFile.write(s)
560 if not g.unitTesting:
561 g.es("created:", newFileName)
562 except Exception:
563 g.es("exception creating:", newFileName)
564 g.print_exception()
565 #@-<< Write s into newFileName >>
566 return None
567 #@+node:ekr.20031218072017.3303: *4* ic.removeSentinelLines
568 # This does not handle @nonl properly, but that no longer matters.
570 def removeSentinelLines(self, s, line_delim, start_delim, unused_end_delim):
571 """Properly remove all sentinle lines in s."""
572 delim = (line_delim or start_delim or '') + '@'
573 verbatim = delim + 'verbatim'
574 verbatimFlag = False
575 result = []
576 for line in g.splitLines(s):
577 i = g.skip_ws(line, 0)
578 if not verbatimFlag and g.match(line, i, delim):
579 if g.match(line, i, verbatim):
580 # Force the next line to be in the result.
581 verbatimFlag = True
582 else:
583 result.append(line)
584 verbatimFlag = False
585 return ''.join(result)
586 #@+node:ekr.20031218072017.1464: *4* ic.weave
587 def weave(self, filename):
588 p = self.c.p
589 nl = self.output_newline
590 if not p:
591 return
592 self.setEncoding()
593 try:
594 with open(filename, 'w', encoding=self.encoding) as f:
595 for p in p.self_and_subtree():
596 s = p.b
597 s2 = s.strip()
598 if s2:
599 f.write("-" * 60)
600 f.write(nl)
601 #@+<< write the context of p to f >>
602 #@+node:ekr.20031218072017.1465: *5* << write the context of p to f >> (weave)
603 # write the headlines of p, p's parent and p's grandparent.
604 context = []
605 p2 = p.copy()
606 i = 0
607 while i < 3:
608 i += 1
609 if not p2:
610 break
611 context.append(p2.h)
612 p2.moveToParent()
613 context.reverse()
614 indent = ""
615 for line in context:
616 f.write(indent)
617 indent += '\t'
618 f.write(line)
619 f.write(nl)
620 #@-<< write the context of p to f >>
621 f.write("-" * 60)
622 f.write(nl)
623 f.write(s.rstrip() + nl)
624 except Exception:
625 g.es("exception opening:", filename)
626 g.print_exception()
627 #@+node:ekr.20031218072017.3209: *3* ic.Import
628 #@+node:ekr.20031218072017.3210: *4* ic.createOutline & helpers
629 def createOutline(self, parent, ext=None, s=None):
630 """
631 Create an outline by importing a file, reading the file with the
632 given encoding if string s is None.
634 ext, The file extension to be used, or None.
635 fileName: A string or None. The name of the file to be read.
636 parent: The parent position of the created outline.
637 s: A string or None. The file's contents.
638 """
639 c = self.c
640 p = parent.copy()
641 self.treeType = '@file' # Fix #352.
642 fileName = g.fullPath(c, parent)
643 if g.is_binary_external_file(fileName):
644 return self.import_binary_file(fileName, parent)
645 # Init ivars.
646 self.setEncoding(
647 p=parent,
648 default=c.config.default_at_auto_file_encoding,
649 )
650 ext, s = self.init_import(ext, fileName, s)
651 if s is None:
652 return None
653 # The so-called scanning func is a callback. It must have a c argument.
654 func = self.dispatch(ext, p)
655 # Call the scanning function.
656 if g.unitTesting:
657 assert func or ext in ('.txt', '.w', '.xxx'), (repr(func), ext, p.h)
658 if func and not c.config.getBool('suppress-import-parsing', default=False):
659 s = g.toUnicode(s, encoding=self.encoding)
660 s = s.replace('\r', '')
661 # func is a factory that instantiates the importer class.
662 ok = func(c=c, parent=p, s=s)
663 else:
664 # Just copy the file to the parent node.
665 s = g.toUnicode(s, encoding=self.encoding)
666 s = s.replace('\r', '')
667 ok = self.scanUnknownFileType(s, p, ext)
668 if g.unitTesting:
669 return p if ok else None
670 # #488894: unsettling dialog when saving Leo file
671 # #889175: Remember the full fileName.
672 c.atFileCommands.rememberReadPath(fileName, p)
673 p.contract()
674 w = c.frame.body.wrapper
675 w.setInsertPoint(0)
676 w.seeInsertPoint()
677 return p
678 #@+node:ekr.20140724064952.18038: *5* ic.dispatch & helpers
679 def dispatch(self, ext, p):
680 """Return the correct scanner function for p, an @auto node."""
681 # Match the @auto type first, then the file extension.
682 c = self.c
683 return g.app.scanner_for_at_auto(c, p) or g.app.scanner_for_ext(c, ext)
684 #@+node:ekr.20170405191106.1: *5* ic.import_binary_file
685 def import_binary_file(self, fileName, parent):
687 # Fix bug 1185409 importing binary files puts binary content in body editor.
688 # Create an @url node.
689 c = self.c
690 if parent:
691 p = parent.insertAsLastChild()
692 else:
693 p = c.lastTopLevel().insertAfter()
694 p.h = f"@url file://{fileName}"
695 return p
696 #@+node:ekr.20140724175458.18052: *5* ic.init_import
697 def init_import(self, ext, fileName, s):
698 """
699 Init ivars imports and read the file into s.
700 Return ext, s.
701 """
702 junk, self.fileName = g.os_path_split(fileName)
703 self.methodName, self.fileType = g.os_path_splitext(self.fileName)
704 if not ext:
705 ext = self.fileType
706 ext = ext.lower()
707 if not s:
708 # Set the kind for error messages in readFileIntoString.
709 s, e = g.readFileIntoString(fileName, encoding=self.encoding)
710 if s is None:
711 return None, None
712 if e:
713 self.encoding = e
714 return ext, s
715 #@+node:ekr.20070713075352: *5* ic.scanUnknownFileType & helper
716 def scanUnknownFileType(self, s, p, ext):
717 """Scan the text of an unknown file type."""
718 body = ''
719 if ext in ('.html', '.htm'):
720 body += '@language html\n'
721 elif ext in ('.txt', '.text'):
722 body += '@nocolor\n'
723 else:
724 language = self.languageForExtension(ext)
725 if language:
726 body += f"@language {language}\n"
727 self.setBodyString(p, body + s)
728 for p in p.self_and_subtree():
729 p.clearDirty()
730 return True
731 #@+node:ekr.20080811174246.1: *6* ic.languageForExtension
732 def languageForExtension(self, ext):
733 """Return the language corresponding to the extension ext."""
734 unknown = 'unknown_language'
735 if ext.startswith('.'):
736 ext = ext[1:]
737 if ext:
738 z = g.app.extra_extension_dict.get(ext)
739 if z not in (None, 'none', 'None'):
740 language = z
741 else:
742 language = g.app.extension_dict.get(ext)
743 if language in (None, 'none', 'None'):
744 language = unknown
745 else:
746 language = unknown
747 # Return the language even if there is no colorizer mode for it.
748 return language
749 #@+node:ekr.20070806111212: *4* ic.readAtAutoNodes
750 def readAtAutoNodes(self):
751 c, p = self.c, self.c.p
752 after = p.nodeAfterTree()
753 found = False
754 while p and p != after:
755 if p.isAtAutoNode():
756 if p.isAtIgnoreNode():
757 g.warning('ignoring', p.h)
758 p.moveToThreadNext()
759 else:
760 c.atFileCommands.readOneAtAutoNode(p)
761 found = True
762 p.moveToNodeAfterTree()
763 else:
764 p.moveToThreadNext()
765 if not g.unitTesting:
766 message = 'finished' if found else 'no @auto nodes in the selected tree'
767 g.blue(message)
768 c.redraw()
769 #@+node:ekr.20031218072017.1810: *4* ic.importDerivedFiles
770 def importDerivedFiles(self, parent=None, paths=None, command='Import'):
771 """
772 Import one or more external files.
773 This is not a command. It must *not* have an event arg.
774 command is None when importing from the command line.
775 """
776 at, c, u = self.c.atFileCommands, self.c, self.c.undoer
777 current = c.p or c.rootPosition()
778 self.tab_width = c.getTabWidth(current)
779 if not paths:
780 return None
781 # Initial open from command line is not undoable.
782 if command:
783 u.beforeChangeGroup(current, command)
784 for fileName in paths:
785 fileName = fileName.replace('\\', '/') # 2011/10/09.
786 g.setGlobalOpenDir(fileName)
787 isThin = at.scanHeaderForThin(fileName)
788 if command:
789 undoData = u.beforeInsertNode(parent)
790 p = parent.insertAfter()
791 if isThin:
792 # Create @file node, not a deprecated @thin node.
793 p.initHeadString("@file " + fileName)
794 at.read(p)
795 else:
796 p.initHeadString("Imported @file " + fileName)
797 at.read(p)
798 p.contract()
799 p.setDirty() # 2011/10/09: tell why the file is dirty!
800 if command:
801 u.afterInsertNode(p, command, undoData)
802 current.expand()
803 c.setChanged()
804 if command:
805 u.afterChangeGroup(p, command)
806 c.redraw(current)
807 return p
808 #@+node:ekr.20031218072017.3212: *4* ic.importFilesCommand
809 def importFilesCommand(self,
810 files=None,
811 parent=None,
812 shortFn=False,
813 treeType=None,
814 verbose=True, # Legacy value.
815 ):
816 # Not a command. It must *not* have an event arg.
817 c, u = self.c, self.c.undoer
818 if not c or not c.p or not files:
819 return
820 self.tab_width = c.getTabWidth(c.p)
821 self.treeType = treeType or '@file'
822 self.verbose = verbose
823 if not parent:
824 g.trace('===== no parent', g.callers())
825 return
826 for fn in files or []:
827 # Report exceptions here, not in the caller.
828 try:
829 g.setGlobalOpenDir(fn)
830 # Leo 5.6: Handle undo here, not in createOutline.
831 undoData = u.beforeInsertNode(parent)
832 p = parent.insertAsLastChild()
833 p.h = f"{treeType} {fn}"
834 u.afterInsertNode(p, 'Import', undoData)
835 p = self.createOutline(parent=p)
836 if p: # createOutline may fail.
837 if self.verbose and not g.unitTesting:
838 g.blue("imported", g.shortFileName(fn) if shortFn else fn)
839 p.contract()
840 p.setDirty()
841 c.setChanged()
842 except Exception:
843 g.es_print('Exception importing', fn)
844 g.es_exception()
845 c.validateOutline()
846 parent.expand()
847 #@+node:ekr.20160503125237.1: *4* ic.importFreeMind
848 def importFreeMind(self, files):
849 """
850 Import a list of .mm.html files exported from FreeMind:
851 http://freemind.sourceforge.net/wiki/index.php/Main_Page
852 """
853 FreeMindImporter(self.c).import_files(files)
854 #@+node:ekr.20160503125219.1: *4* ic.importMindMap
855 def importMindMap(self, files):
856 """
857 Import a list of .csv files exported from MindJet:
858 https://www.mindjet.com/
859 """
860 MindMapImporter(self.c).import_files(files)
861 #@+node:ekr.20031218072017.3224: *4* ic.importWebCommand & helpers
862 def importWebCommand(self, files, webType):
863 c, current = self.c, self.c.p
864 if current is None:
865 return
866 if not files:
867 return
868 self.tab_width = c.getTabWidth(current) # New in 4.3.
869 self.webType = webType
870 for fileName in files:
871 g.setGlobalOpenDir(fileName)
872 p = self.createOutlineFromWeb(fileName, current)
873 p.contract()
874 p.setDirty()
875 c.setChanged()
876 c.redraw(current)
877 #@+node:ekr.20031218072017.3225: *5* createOutlineFromWeb
878 def createOutlineFromWeb(self, path, parent):
879 c = self.c
880 u = c.undoer
881 junk, fileName = g.os_path_split(path)
882 undoData = u.beforeInsertNode(parent)
883 # Create the top-level headline.
884 p = parent.insertAsLastChild()
885 p.initHeadString(fileName)
886 if self.webType == "cweb":
887 self.setBodyString(p, "@ignore\n@language cweb")
888 # Scan the file, creating one section for each function definition.
889 self.scanWebFile(path, p)
890 u.afterInsertNode(p, 'Import', undoData)
891 return p
892 #@+node:ekr.20031218072017.3227: *5* findFunctionDef
893 def findFunctionDef(self, s, i):
894 # Look at the next non-blank line for a function name.
895 i = g.skip_ws_and_nl(s, i)
896 k = g.skip_line(s, i)
897 name = None
898 while i < k:
899 if g.is_c_id(s[i]):
900 j = i
901 i = g.skip_c_id(s, i)
902 name = s[j:i]
903 elif s[i] == '(':
904 if name:
905 return name
906 break
907 else:
908 i += 1
909 return None
910 #@+node:ekr.20031218072017.3228: *5* scanBodyForHeadline
911 #@+at This method returns the proper headline text.
912 # 1. If s contains a section def, return the section ref.
913 # 2. cweb only: if s contains @c, return the function name following the @c.
914 # 3. cweb only: if s contains @d name, returns @d name.
915 # 4. Otherwise, returns "@"
916 #@@c
918 def scanBodyForHeadline(self, s):
919 if self.webType == "cweb":
920 #@+<< scan cweb body for headline >>
921 #@+node:ekr.20031218072017.3229: *6* << scan cweb body for headline >>
922 i = 0
923 while i < len(s):
924 i = g.skip_ws_and_nl(s, i)
925 # Allow constructs such as @ @c, or @ @<.
926 if self.isDocStart(s, i):
927 i += 2
928 i = g.skip_ws(s, i)
929 if g.match(s, i, "@d") or g.match(s, i, "@f"):
930 # Look for a macro name.
931 directive = s[i : i + 2]
932 i = g.skip_ws(s, i + 2) # skip the @d or @f
933 if i < len(s) and g.is_c_id(s[i]):
934 j = i
935 g.skip_c_id(s, i)
936 return s[j:i]
937 return directive
938 if g.match(s, i, "@c") or g.match(s, i, "@p"):
939 # Look for a function def.
940 name = self.findFunctionDef(s, i + 2)
941 return name if name else "outer function"
942 if g.match(s, i, "@<"):
943 # Look for a section def.
944 # A small bug: the section def must end on this line.
945 j = i
946 k = g.find_on_line(s, i, "@>")
947 if k > -1 and (g.match(s, k + 2, "+=") or g.match(s, k + 2, "=")):
948 return s[j : k + 2] # return the section ref.
949 i = g.skip_line(s, i)
950 #@-<< scan cweb body for headline >>
951 else:
952 #@+<< scan noweb body for headline >>
953 #@+node:ekr.20031218072017.3230: *6* << scan noweb body for headline >>
954 i = 0
955 while i < len(s):
956 i = g.skip_ws_and_nl(s, i)
957 if g.match(s, i, "<<"):
958 k = g.find_on_line(s, i, ">>=")
959 if k > -1:
960 ref = s[i : k + 2]
961 name = s[i + 2 : k].strip()
962 if name != "@others":
963 return ref
964 else:
965 name = self.findFunctionDef(s, i)
966 if name:
967 return name
968 i = g.skip_line(s, i)
969 #@-<< scan noweb body for headline >>
970 return "@" # default.
971 #@+node:ekr.20031218072017.3231: *5* scanWebFile (handles limbo)
972 def scanWebFile(self, fileName, parent):
973 theType = self.webType
974 lb = "@<" if theType == "cweb" else "<<"
975 rb = "@>" if theType == "cweb" else ">>"
976 s, e = g.readFileIntoString(fileName)
977 if s is None:
978 return
979 #@+<< Create a symbol table of all section names >>
980 #@+node:ekr.20031218072017.3232: *6* << Create a symbol table of all section names >>
981 i = 0
982 self.web_st = []
983 while i < len(s):
984 progress = i
985 i = g.skip_ws_and_nl(s, i)
986 if self.isDocStart(s, i):
987 if theType == "cweb":
988 i += 2
989 else:
990 i = g.skip_line(s, i)
991 elif theType == "cweb" and g.match(s, i, "@@"):
992 i += 2
993 elif g.match(s, i, lb):
994 i += 2
995 j = i
996 k = g.find_on_line(s, j, rb)
997 if k > -1:
998 self.cstEnter(s[j:k])
999 else:
1000 i += 1
1001 assert i > progress
1002 #@-<< Create a symbol table of all section names >>
1003 #@+<< Create nodes for limbo text and the root section >>
1004 #@+node:ekr.20031218072017.3233: *6* << Create nodes for limbo text and the root section >>
1005 i = 0
1006 while i < len(s):
1007 progress = i
1008 i = g.skip_ws_and_nl(s, i)
1009 if self.isModuleStart(s, i) or g.match(s, i, lb):
1010 break
1011 else: i = g.skip_line(s, i)
1012 assert i > progress
1013 j = g.skip_ws(s, 0)
1014 if j < i:
1015 self.createHeadline(parent, "@ " + s[j:i], "Limbo")
1016 j = i
1017 if g.match(s, i, lb):
1018 while i < len(s):
1019 progress = i
1020 i = g.skip_ws_and_nl(s, i)
1021 if self.isModuleStart(s, i):
1022 break
1023 else: i = g.skip_line(s, i)
1024 assert i > progress
1025 self.createHeadline(parent, s[j:i], g.angleBrackets(" @ "))
1027 #@-<< Create nodes for limbo text and the root section >>
1028 while i < len(s):
1029 outer_progress = i
1030 #@+<< Create a node for the next module >>
1031 #@+node:ekr.20031218072017.3234: *6* << Create a node for the next module >>
1032 if theType == "cweb":
1033 assert self.isModuleStart(s, i)
1034 start = i
1035 if self.isDocStart(s, i):
1036 i += 2
1037 while i < len(s):
1038 progress = i
1039 i = g.skip_ws_and_nl(s, i)
1040 if self.isModuleStart(s, i):
1041 break
1042 else:
1043 i = g.skip_line(s, i)
1044 assert i > progress
1045 #@+<< Handle cweb @d, @f, @c and @p directives >>
1046 #@+node:ekr.20031218072017.3235: *7* << Handle cweb @d, @f, @c and @p directives >>
1047 if g.match(s, i, "@d") or g.match(s, i, "@f"):
1048 i += 2
1049 i = g.skip_line(s, i)
1050 # Place all @d and @f directives in the same node.
1051 while i < len(s):
1052 progress = i
1053 i = g.skip_ws_and_nl(s, i)
1054 if g.match(s, i, "@d") or g.match(s, i, "@f"):
1055 i = g.skip_line(s, i)
1056 else:
1057 break
1058 assert i > progress
1059 i = g.skip_ws_and_nl(s, i)
1060 while i < len(s) and not self.isModuleStart(s, i):
1061 progress = i
1062 i = g.skip_line(s, i)
1063 i = g.skip_ws_and_nl(s, i)
1064 assert i > progress
1065 if g.match(s, i, "@c") or g.match(s, i, "@p"):
1066 i += 2
1067 while i < len(s):
1068 progress = i
1069 i = g.skip_line(s, i)
1070 i = g.skip_ws_and_nl(s, i)
1071 if self.isModuleStart(s, i):
1072 break
1073 assert i > progress
1074 #@-<< Handle cweb @d, @f, @c and @p directives >>
1075 else:
1076 assert self.isDocStart(s, i)
1077 start = i
1078 i = g.skip_line(s, i)
1079 while i < len(s):
1080 progress = i
1081 i = g.skip_ws_and_nl(s, i)
1082 if self.isDocStart(s, i):
1083 break
1084 else:
1085 i = g.skip_line(s, i)
1086 assert i > progress
1087 body = s[start:i]
1088 body = self.massageWebBody(body)
1089 headline = self.scanBodyForHeadline(body)
1090 self.createHeadline(parent, body, headline)
1091 #@-<< Create a node for the next module >>
1092 assert i > outer_progress
1093 #@+node:ekr.20031218072017.3236: *5* Symbol table
1094 #@+node:ekr.20031218072017.3237: *6* cstCanonicalize
1095 # We canonicalize strings before looking them up,
1096 # but strings are entered in the form they are first encountered.
1098 def cstCanonicalize(self, s, lower=True):
1099 if lower:
1100 s = s.lower()
1101 s = s.replace("\t", " ").replace("\r", "")
1102 s = s.replace("\n", " ").replace(" ", " ")
1103 return s.strip()
1104 #@+node:ekr.20031218072017.3238: *6* cstDump
1105 def cstDump(self):
1106 s = "Web Symbol Table...\n\n"
1107 for name in sorted(self.web_st):
1108 s += name + "\n"
1109 return s
1110 #@+node:ekr.20031218072017.3239: *6* cstEnter
1111 # We only enter the section name into the symbol table if the ... convention is not used.
1113 def cstEnter(self, s):
1114 # Don't enter names that end in "..."
1115 s = s.rstrip()
1116 if s.endswith("..."):
1117 return
1118 # Put the section name in the symbol table, retaining capitalization.
1119 lower = self.cstCanonicalize(s, True) # do lower
1120 upper = self.cstCanonicalize(s, False) # don't lower.
1121 for name in self.web_st:
1122 if name.lower() == lower:
1123 return
1124 self.web_st.append(upper)
1125 #@+node:ekr.20031218072017.3240: *6* cstLookup
1126 # This method returns a string if the indicated string is a prefix of an entry in the web_st.
1128 def cstLookup(self, target):
1129 # Do nothing if the ... convention is not used.
1130 target = target.strip()
1131 if not target.endswith("..."):
1132 return target
1133 # Canonicalize the target name, and remove the trailing "..."
1134 ctarget = target[:-3]
1135 ctarget = self.cstCanonicalize(ctarget).strip()
1136 found = False
1137 result = target
1138 for s in self.web_st:
1139 cs = self.cstCanonicalize(s)
1140 if cs[: len(ctarget)] == ctarget:
1141 if found:
1142 g.es('', f"****** {target}", 'is also a prefix of', s)
1143 else:
1144 found = True
1145 result = s
1146 # g.es("replacing",target,"with",s)
1147 return result
1148 #@+node:ekr.20140531104908.18833: *3* ic.parse_body
1149 def parse_body(self, p):
1150 """
1151 Parse p.b as source code, creating a tree of descendant nodes.
1152 This is essentially an import of p.b.
1153 """
1154 if not p:
1155 return
1156 c, d, ic = self.c, g.app.language_extension_dict, self
1157 if p.hasChildren():
1158 g.es_print('can not run parse-body: node has children:', p.h)
1159 return
1160 language = g.scanForAtLanguage(c, p)
1161 self.treeType = '@file'
1162 ext = '.' + d.get(language)
1163 parser = g.app.classDispatchDict.get(ext)
1164 # Fix bug 151: parse-body creates "None declarations"
1165 if p.isAnyAtFileNode():
1166 fn = p.anyAtFileNodeName()
1167 ic.methodName, ic.fileType = g.os_path_splitext(fn)
1168 else:
1169 fileType = d.get(language, 'py')
1170 ic.methodName, ic.fileType = p.h, fileType
1171 if not parser:
1172 g.es_print(f"parse-body: no parser for @language {language or 'None'}")
1173 return
1174 bunch = c.undoer.beforeChangeTree(p)
1175 s = p.b
1176 p.b = ''
1177 try:
1178 parser(c, s, p) # 2357.
1179 c.undoer.afterChangeTree(p, 'parse-body', bunch)
1180 p.expand()
1181 c.selectPosition(p)
1182 c.redraw()
1183 except Exception:
1184 g.es_exception()
1185 p.b = s
1186 #@+node:ekr.20031218072017.3305: *3* ic.Utilities
1187 #@+node:ekr.20090122201952.4: *4* ic.appendStringToBody & setBodyString (leoImport)
1188 def appendStringToBody(self, p, s):
1189 """Similar to c.appendStringToBody,
1190 but does not recolor the text or redraw the screen."""
1191 if s:
1192 p.b = p.b + g.toUnicode(s, self.encoding)
1194 def setBodyString(self, p, s):
1195 """
1196 Similar to c.setBodyString, but does not recolor the text or
1197 redraw the screen.
1198 """
1199 c, v = self.c, p.v
1200 if not c or not p:
1201 return
1202 s = g.toUnicode(s, self.encoding)
1203 if c.p and p.v == c.p.v:
1204 w = c.frame.body.wrapper
1205 i = len(s)
1206 w.setAllText(s)
1207 w.setSelectionRange(i, i, insert=i)
1208 # Keep the body text up-to-date.
1209 if v.b != s:
1210 v.setBodyString(s)
1211 v.setSelection(0, 0)
1212 p.setDirty()
1213 if not c.isChanged():
1214 c.setChanged()
1215 #@+node:ekr.20031218072017.3306: *4* ic.createHeadline
1216 def createHeadline(self, parent, body, headline):
1217 """Create a new VNode as the last child of parent position."""
1218 p = parent.insertAsLastChild()
1219 p.initHeadString(headline)
1220 if body:
1221 self.setBodyString(p, body)
1222 return p
1223 #@+node:ekr.20031218072017.3307: *4* ic.error
1224 def error(self, s):
1225 g.es('', s)
1226 #@+node:ekr.20031218072017.3309: *4* ic.isDocStart & isModuleStart
1227 # The start of a document part or module in a noweb or cweb file.
1228 # Exporters may have to test for @doc as well.
1230 def isDocStart(self, s, i):
1231 if not g.match(s, i, "@"):
1232 return False
1233 j = g.skip_ws(s, i + 1)
1234 if g.match(s, j, "%defs"):
1235 return False
1236 if self.webType == "cweb" and g.match(s, i, "@*"):
1237 return True
1238 return g.match(s, i, "@ ") or g.match(s, i, "@\t") or g.match(s, i, "@\n")
1240 def isModuleStart(self, s, i):
1241 if self.isDocStart(s, i):
1242 return True
1243 return self.webType == "cweb" and (
1244 g.match(s, i, "@c") or g.match(s, i, "@p") or
1245 g.match(s, i, "@d") or g.match(s, i, "@f"))
1246 #@+node:ekr.20031218072017.3312: *4* ic.massageWebBody
1247 def massageWebBody(self, s):
1248 theType = self.webType
1249 lb = "@<" if theType == "cweb" else "<<"
1250 rb = "@>" if theType == "cweb" else ">>"
1251 #@+<< Remove most newlines from @space and @* sections >>
1252 #@+node:ekr.20031218072017.3313: *5* << Remove most newlines from @space and @* sections >>
1253 i = 0
1254 while i < len(s):
1255 progress = i
1256 i = g.skip_ws_and_nl(s, i)
1257 if self.isDocStart(s, i):
1258 # Scan to end of the doc part.
1259 if g.match(s, i, "@ %def"):
1260 # Don't remove the newline following %def
1261 i = g.skip_line(s, i)
1262 start = end = i
1263 else:
1264 start = end = i
1265 i += 2
1266 while i < len(s):
1267 progress2 = i
1268 i = g.skip_ws_and_nl(s, i)
1269 if self.isModuleStart(s, i) or g.match(s, i, lb):
1270 end = i
1271 break
1272 elif theType == "cweb":
1273 i += 1
1274 else:
1275 i = g.skip_to_end_of_line(s, i)
1276 assert i > progress2
1277 # Remove newlines from start to end.
1278 doc = s[start:end]
1279 doc = doc.replace("\n", " ")
1280 doc = doc.replace("\r", "")
1281 doc = doc.strip()
1282 if doc:
1283 if doc == "@":
1284 doc = "@ " if self.webType == "cweb" else "@\n"
1285 else:
1286 doc += "\n\n"
1287 s = s[:start] + doc + s[end:]
1288 i = start + len(doc)
1289 else: i = g.skip_line(s, i)
1290 assert i > progress
1291 #@-<< Remove most newlines from @space and @* sections >>
1292 #@+<< Replace abbreviated names with full names >>
1293 #@+node:ekr.20031218072017.3314: *5* << Replace abbreviated names with full names >>
1294 i = 0
1295 while i < len(s):
1296 progress = i
1297 if g.match(s, i, lb):
1298 i += 2
1299 j = i
1300 k = g.find_on_line(s, j, rb)
1301 if k > -1:
1302 name = s[j:k]
1303 name2 = self.cstLookup(name)
1304 if name != name2:
1305 # Replace name by name2 in s.
1306 s = s[:j] + name2 + s[k:]
1307 i = j + len(name2)
1308 i = g.skip_line(s, i)
1309 assert i > progress
1310 #@-<< Replace abbreviated names with full names >>
1311 s = s.rstrip()
1312 return s
1313 #@+node:ekr.20031218072017.1463: *4* ic.setEncoding
1314 def setEncoding(self, p=None, default=None):
1315 c = self.c
1316 encoding = g.getEncodingAt(p or c.p) or default
1317 if encoding and g.isValidEncoding(encoding):
1318 self.encoding = encoding
1319 elif default:
1320 self.encoding = default
1321 else:
1322 self.encoding = 'utf-8'
1323 #@-others
1324#@+node:ekr.20160503144404.1: ** class MindMapImporter
1325class MindMapImporter:
1326 """Mind Map Importer class."""
1328 def __init__(self, c):
1329 """ctor for MindMapImporter class."""
1330 self.c = c
1331 #@+others
1332 #@+node:ekr.20160503130209.1: *3* mindmap.create_outline
1333 def create_outline(self, path):
1334 c = self.c
1335 junk, fileName = g.os_path_split(path)
1336 undoData = c.undoer.beforeInsertNode(c.p)
1337 # Create the top-level headline.
1338 p = c.lastTopLevel().insertAfter()
1339 fn = g.shortFileName(path).strip()
1340 if fn.endswith('.csv'):
1341 fn = fn[:-4]
1342 p.h = fn
1343 try:
1344 self.scan(path, p)
1345 except Exception:
1346 g.es_print('Invalid MindJet file:', fn)
1347 c.undoer.afterInsertNode(p, 'Import', undoData)
1348 return p
1349 #@+node:ekr.20160503144647.1: *3* mindmap.import_files
1350 def import_files(self, files):
1351 """Import a list of MindMap (.csv) files."""
1352 c = self.c
1353 if files:
1354 self.tab_width = c.getTabWidth(c.p)
1355 for fileName in files:
1356 g.setGlobalOpenDir(fileName)
1357 p = self.create_outline(fileName)
1358 p.contract()
1359 p.setDirty()
1360 c.setChanged()
1361 c.redraw(p)
1362 #@+node:ekr.20160504043243.1: *3* mindmap.prompt_for_files
1363 def prompt_for_files(self):
1364 """Prompt for a list of MindJet (.csv) files and import them."""
1365 c = self.c
1366 types = [
1367 ("MindJet files", "*.csv"),
1368 ("All files", "*"),
1369 ]
1370 names = g.app.gui.runOpenFileDialog(c,
1371 title="Import MindJet File",
1372 filetypes=types,
1373 defaultextension=".csv",
1374 multiple=True)
1375 c.bringToFront()
1376 if names:
1377 g.chdir(names[0])
1378 self.import_files(names)
1379 #@+node:ekr.20160503130256.1: *3* mindmap.scan & helpers
1380 def scan(self, path, target):
1381 """Create an outline from a MindMap (.csv) file."""
1382 c = self.c
1383 f = open(path)
1384 reader = csv.reader(f)
1385 max_chars_in_header = 80
1386 n1 = n = target.level()
1387 p = target.copy()
1388 for row in list(reader)[1:]:
1389 new_level = self.csv_level(row) + n1
1390 self.csv_string(row)
1391 if new_level > n:
1392 p = p.insertAsLastChild().copy()
1393 p.b = self.csv_string(row)
1394 n = n + 1
1395 elif new_level == n:
1396 p = p.insertAfter().copy()
1397 p.b = self.csv_string(row)
1398 elif new_level < n:
1399 for item in p.parents():
1400 if item.level() == new_level - 1:
1401 p = item.copy()
1402 break
1403 p = p.insertAsLastChild().copy()
1404 p.b = self.csv_string(row)
1405 n = p.level()
1406 for p in target.unique_subtree():
1407 if len(p.b.splitlines()) == 1:
1408 if len(p.b.splitlines()[0]) < max_chars_in_header:
1409 p.h = p.b.splitlines()[0]
1410 p.b = ""
1411 else:
1412 p.h = "@node_with_long_text"
1413 else:
1414 p.h = "@node_with_long_text"
1415 c.redraw()
1416 f.close()
1417 #@+node:ekr.20160503130810.4: *4* mindmap.csv_level
1418 def csv_level(self, row):
1419 """Return the level of the given row."""
1420 count = 0
1421 while count <= len(row):
1422 if row[count]:
1423 return count + 1
1424 count = count + 1
1425 return -1
1426 #@+node:ekr.20160503130810.5: *4* mindmap.csv_string
1427 def csv_string(self, row):
1428 """Return the string for the given csv row."""
1429 count = 0
1430 while count <= len(row):
1431 if row[count]:
1432 return row[count]
1433 count = count + 1
1434 return None
1435 #@-others
1436#@+node:ekr.20161006100941.1: ** class MORE_Importer
1437class MORE_Importer:
1438 """Class to import MORE files."""
1440 def __init__(self, c):
1441 """ctor for MORE_Importer class."""
1442 self.c = c
1443 #@+others
1444 #@+node:ekr.20161006101111.1: *3* MORE.prompt_for_files
1445 def prompt_for_files(self):
1446 """Prompt for a list of MORE files and import them."""
1447 c = self.c
1448 types = [
1449 ("All files", "*"),
1450 ]
1451 names = g.app.gui.runOpenFileDialog(c,
1452 title="Import MORE Files",
1453 filetypes=types,
1454 # defaultextension=".txt",
1455 multiple=True)
1456 c.bringToFront()
1457 if names:
1458 g.chdir(names[0])
1459 self.import_files(names)
1460 #@+node:ekr.20161006101218.1: *3* MORE.import_files
1461 def import_files(self, files):
1462 """Import a list of MORE (.csv) files."""
1463 c = self.c
1464 if files:
1465 changed = False
1466 self.tab_width = c.getTabWidth(c.p)
1467 for fileName in files:
1468 g.setGlobalOpenDir(fileName)
1469 p = self.import_file(fileName)
1470 if p:
1471 p.contract()
1472 p.setDirty()
1473 c.setChanged()
1474 changed = True
1475 if changed:
1476 c.redraw(p)
1477 #@+node:ekr.20161006101347.1: *3* MORE.import_file
1478 def import_file(self, fileName): # Not a command, so no event arg.
1479 c = self.c
1480 u = c.undoer
1481 ic = c.importCommands
1482 if not c.p:
1483 return None
1484 ic.setEncoding()
1485 g.setGlobalOpenDir(fileName)
1486 s, e = g.readFileIntoString(fileName)
1487 if s is None:
1488 return None
1489 s = s.replace('\r', '') # Fixes bug 626101.
1490 lines = g.splitLines(s)
1491 # Convert the string to an outline and insert it after the current node.
1492 if self.check_lines(lines):
1493 last = c.lastTopLevel()
1494 undoData = u.beforeInsertNode(c.p)
1495 root = last.insertAfter()
1496 root.h = fileName
1497 p = self.import_lines(lines, root)
1498 if p:
1499 c.endEditing()
1500 c.validateOutline()
1501 p.setDirty()
1502 c.setChanged()
1503 u.afterInsertNode(root, 'Import MORE File', undoData)
1504 c.selectPosition(root)
1505 c.redraw()
1506 return root
1507 if not g.unitTesting:
1508 g.es("not a valid MORE file", fileName)
1509 return None
1510 #@+node:ekr.20031218072017.3215: *3* MORE.import_lines
1511 def import_lines(self, strings, first_p):
1512 c = self.c
1513 if not strings:
1514 return None
1515 if not self.check_lines(strings):
1516 return None
1517 firstLevel, junk = self.headlineLevel(strings[0])
1518 lastLevel = -1
1519 theRoot = last_p = None
1520 index = 0
1521 while index < len(strings):
1522 progress = index
1523 s = strings[index]
1524 level, junk = self.headlineLevel(s)
1525 level -= firstLevel
1526 if level >= 0:
1527 #@+<< Link a new position p into the outline >>
1528 #@+node:ekr.20031218072017.3216: *4* << Link a new position p into the outline >>
1529 assert level >= 0
1530 if not last_p:
1531 theRoot = p = first_p.insertAsLastChild() # 2016/10/06.
1532 elif level == lastLevel:
1533 p = last_p.insertAfter()
1534 elif level == lastLevel + 1:
1535 p = last_p.insertAsNthChild(0)
1536 else:
1537 assert level < lastLevel
1538 while level < lastLevel:
1539 lastLevel -= 1
1540 last_p = last_p.parent()
1541 assert last_p
1542 assert lastLevel >= 0
1543 p = last_p.insertAfter()
1544 last_p = p
1545 lastLevel = level
1546 #@-<< Link a new position p into the outline >>
1547 #@+<< Set the headline string, skipping over the leader >>
1548 #@+node:ekr.20031218072017.3217: *4* << Set the headline string, skipping over the leader >>
1549 j = 0
1550 while g.match(s, j, '\t') or g.match(s, j, ' '):
1551 j += 1
1552 if g.match(s, j, "+ ") or g.match(s, j, "- "):
1553 j += 2
1554 p.initHeadString(s[j:])
1555 #@-<< Set the headline string, skipping over the leader >>
1556 #@+<< Count the number of following body lines >>
1557 #@+node:ekr.20031218072017.3218: *4* << Count the number of following body lines >>
1558 bodyLines = 0
1559 index += 1 # Skip the headline.
1560 while index < len(strings):
1561 s = strings[index]
1562 level, junk = self.headlineLevel(s)
1563 level -= firstLevel
1564 if level >= 0:
1565 break
1566 # Remove first backslash of the body line.
1567 if g.match(s, 0, '\\'):
1568 strings[index] = s[1:]
1569 bodyLines += 1
1570 index += 1
1571 #@-<< Count the number of following body lines >>
1572 #@+<< Add the lines to the body text of p >>
1573 #@+node:ekr.20031218072017.3219: *4* << Add the lines to the body text of p >>
1574 if bodyLines > 0:
1575 body = ""
1576 n = index - bodyLines
1577 while n < index:
1578 body += strings[n].rstrip()
1579 if n != index - 1:
1580 body += "\n"
1581 n += 1
1582 p.setBodyString(body)
1583 #@-<< Add the lines to the body text of p >>
1584 p.setDirty()
1585 else: index += 1
1586 assert progress < index
1587 if theRoot:
1588 theRoot.setDirty()
1589 c.setChanged()
1590 c.redraw()
1591 return theRoot
1592 #@+node:ekr.20031218072017.3222: *3* MORE.headlineLevel
1593 def headlineLevel(self, s):
1594 """return the headline level of s,or -1 if the string is not a MORE headline."""
1595 level = 0
1596 i = 0
1597 while i < len(s) and s[i] in ' \t': # 2016/10/06: allow blanks or tabs.
1598 level += 1
1599 i += 1
1600 plusFlag = g.match(s, i, "+")
1601 if g.match(s, i, "+ ") or g.match(s, i, "- "):
1602 return level, plusFlag
1603 return -1, plusFlag
1604 #@+node:ekr.20031218072017.3223: *3* MORE.check & check_lines
1605 def check(self, s):
1606 s = s.replace("\r", "")
1607 strings = g.splitLines(s)
1608 return self.check_lines(strings)
1610 def check_lines(self, strings):
1612 if not strings:
1613 return False
1614 level1, plusFlag = self.headlineLevel(strings[0])
1615 if level1 == -1:
1616 return False
1617 # Check the level of all headlines.
1618 lastLevel = level1
1619 for s in strings:
1620 level, newFlag = self.headlineLevel(s)
1621 if level == -1:
1622 return True # A body line.
1623 if level < level1 or level > lastLevel + 1:
1624 return False # improper level.
1625 if level > lastLevel and not plusFlag:
1626 return False # parent of this node has no children.
1627 if level == lastLevel and plusFlag:
1628 return False # last node has missing child.
1629 lastLevel = level
1630 plusFlag = newFlag
1631 return True
1632 #@-others
1633#@+node:ekr.20130823083943.12596: ** class RecursiveImportController
1634class RecursiveImportController:
1635 """Recursively import all python files in a directory and clean the result."""
1636 #@+others
1637 #@+node:ekr.20130823083943.12615: *3* ric.ctor
1638 def __init__(self, c, kind,
1639 add_context=None, # Override setting only if True/False
1640 add_file_context=None, # Override setting only if True/False
1641 add_path=True,
1642 recursive=True,
1643 safe_at_file=True,
1644 theTypes=None,
1645 ignore_pattern=None,
1646 verbose=True, # legacy value.
1647 ):
1648 """Ctor for RecursiveImportController class."""
1649 self.c = c
1650 self.add_path = add_path
1651 self.file_pattern = re.compile(r'^(@@|@)(auto|clean|edit|file|nosent)')
1652 self.ignore_pattern = ignore_pattern or re.compile(r'\.git|node_modules')
1653 self.kind = kind # in ('@auto', '@clean', '@edit', '@file', '@nosent')
1654 self.recursive = recursive
1655 self.root = None
1656 self.safe_at_file = safe_at_file
1657 self.theTypes = theTypes
1658 self.verbose = verbose
1659 # #1605:
1661 def set_bool(setting, val):
1662 if val not in (True, False):
1663 return
1664 c.config.set(None, 'bool', setting, val, warn=True)
1666 set_bool('add-context-to-headlines', add_context)
1667 set_bool('add-file-context-to-headlines', add_file_context)
1668 #@+node:ekr.20130823083943.12613: *3* ric.run & helpers
1669 def run(self, dir_):
1670 """
1671 Import all files whose extension matches self.theTypes in dir_.
1672 In fact, dir_ can be a path to a single file.
1673 """
1674 if self.kind not in ('@auto', '@clean', '@edit', '@file', '@nosent'):
1675 g.es('bad kind param', self.kind, color='red')
1676 try:
1677 c = self.c
1678 p1 = self.root = c.p
1679 t1 = time.time()
1680 g.app.disable_redraw = True
1681 bunch = c.undoer.beforeChangeTree(p1)
1682 # Leo 5.6: Always create a new last top-level node.
1683 last = c.lastTopLevel()
1684 parent = last.insertAfter()
1685 parent.v.h = 'imported files'
1686 # Leo 5.6: Special case for a single file.
1687 self.n_files = 0
1688 if g.os_path_isfile(dir_):
1689 if self.verbose:
1690 g.es_print('\nimporting file:', dir_)
1691 self.import_one_file(dir_, parent)
1692 else:
1693 self.import_dir(dir_, parent)
1694 self.post_process(parent, dir_)
1695 # Fix # 1033.
1696 c.undoer.afterChangeTree(p1, 'recursive-import', bunch)
1697 except Exception:
1698 g.es_print('Exception in recursive import')
1699 g.es_exception()
1700 finally:
1701 g.app.disable_redraw = False
1702 for p2 in parent.self_and_subtree(copy=False):
1703 p2.contract()
1704 c.redraw(parent)
1705 t2 = time.time()
1706 n = len(list(parent.self_and_subtree()))
1707 g.es_print(
1708 f"imported {n} node{g.plural(n)} "
1709 f"in {self.n_files} file{g.plural(self.n_files)} "
1710 f"in {t2 - t1:2.2f} seconds")
1711 #@+node:ekr.20130823083943.12597: *4* ric.import_dir
1712 def import_dir(self, dir_, parent):
1713 """Import selected files from dir_, a directory."""
1714 if g.os_path_isfile(dir_):
1715 files = [dir_]
1716 else:
1717 if self.verbose:
1718 g.es_print('importing directory:', dir_)
1719 files = os.listdir(dir_)
1720 dirs, files2 = [], []
1721 for path in files:
1722 try:
1723 # Fix #408. Catch path exceptions.
1724 # The idea here is to keep going on small errors.
1725 path = g.os_path_join(dir_, path)
1726 if g.os_path_isfile(path):
1727 name, ext = g.os_path_splitext(path)
1728 if ext in self.theTypes:
1729 files2.append(path)
1730 elif self.recursive:
1731 if not self.ignore_pattern.search(path):
1732 dirs.append(path)
1733 except OSError:
1734 g.es_print('Exception computing', path)
1735 g.es_exception()
1736 if files or dirs:
1737 assert parent and parent.v != self.root.v, g.callers()
1738 parent = parent.insertAsLastChild()
1739 parent.v.h = dir_
1740 if files2:
1741 for f in files2:
1742 if not self.ignore_pattern.search(f):
1743 self.import_one_file(f, parent=parent)
1744 if dirs:
1745 assert self.recursive
1746 for dir_ in sorted(dirs):
1747 self.import_dir(dir_, parent)
1748 #@+node:ekr.20170404103953.1: *4* ric.import_one_file
1749 def import_one_file(self, path, parent):
1750 """Import one file to the last top-level node."""
1751 c = self.c
1752 self.n_files += 1
1753 assert parent and parent.v != self.root.v, g.callers()
1754 if self.kind == '@edit':
1755 p = parent.insertAsLastChild()
1756 p.v.h = '@edit ' + path.replace('\\', '/') # 2021/02/19: bug fix: add @edit.
1757 s, e = g.readFileIntoString(path, kind=self.kind)
1758 p.v.b = s
1759 return
1760 # #1484: Use this for @auto as well.
1761 c.importCommands.importFilesCommand(
1762 files=[path],
1763 parent=parent,
1764 shortFn=True,
1765 treeType='@file', # '@auto','@clean','@nosent' cause problems.
1766 verbose=self.verbose, # Leo 6.6.
1767 )
1768 p = parent.lastChild()
1769 p.h = self.kind + p.h[5:] # Honor the requested kind.
1770 if self.safe_at_file:
1771 p.v.h = '@' + p.v.h
1772 #@+node:ekr.20130823083943.12607: *4* ric.post_process & helpers
1773 def post_process(self, p, prefix):
1774 """
1775 Traverse p's tree, replacing all nodes that start with prefix
1776 by the smallest equivalent @path or @file node.
1777 """
1778 self.fix_back_slashes(p)
1779 prefix = prefix.replace('\\', '/')
1780 if self.kind not in ('@auto', '@edit'):
1781 self.remove_empty_nodes(p)
1782 if p.firstChild():
1783 self.minimize_headlines(p.firstChild(), prefix)
1784 self.clear_dirty_bits(p)
1785 self.add_class_names(p)
1786 #@+node:ekr.20180524100258.1: *5* ric.add_class_names
1787 def add_class_names(self, p):
1788 """Add class names to headlines for all descendant nodes."""
1789 # pylint: disable=no-else-continue
1790 after, class_name = None, None
1791 class_paren_pattern = re.compile(r'(.*)\(.*\)\.(.*)')
1792 paren_pattern = re.compile(r'(.*)\(.*\.py\)')
1793 for p in p.self_and_subtree(copy=False):
1794 # Part 1: update the status.
1795 m = self.file_pattern.match(p.h)
1796 if m:
1797 # prefix = m.group(1)
1798 # fn = g.shortFileName(p.h[len(prefix):].strip())
1799 after, class_name = None, None
1800 continue
1801 elif p.h.startswith('@path '):
1802 after, class_name = None, None
1803 elif p.h.startswith('class '):
1804 class_name = p.h[5:].strip()
1805 if class_name:
1806 after = p.nodeAfterTree()
1807 continue
1808 elif p == after:
1809 after, class_name = None, None
1810 # Part 2: update the headline.
1811 if class_name:
1812 if p.h.startswith(class_name):
1813 m = class_paren_pattern.match(p.h)
1814 if m:
1815 p.h = f"{m.group(1)}.{m.group(2)}".rstrip()
1816 else:
1817 p.h = f"{class_name}.{p.h}"
1818 else:
1819 m = paren_pattern.match(p.h)
1820 if m:
1821 p.h = m.group(1).rstrip()
1822 # elif fn:
1823 # tag = ' (%s)' % fn
1824 # if not p.h.endswith(tag):
1825 # p.h += tag
1826 #@+node:ekr.20130823083943.12608: *5* ric.clear_dirty_bits
1827 def clear_dirty_bits(self, p):
1828 c = self.c
1829 c.clearChanged() # Clears *all* dirty bits.
1830 for p in p.self_and_subtree(copy=False):
1831 p.clearDirty()
1832 #@+node:ekr.20130823083943.12609: *5* ric.dump_headlines
1833 def dump_headlines(self, p):
1834 # show all headlines.
1835 for p in p.self_and_subtree(copy=False):
1836 print(p.h)
1837 #@+node:ekr.20130823083943.12610: *5* ric.fix_back_slashes
1838 def fix_back_slashes(self, p):
1839 """Convert backslash to slash in all headlines."""
1840 for p in p.self_and_subtree(copy=False):
1841 s = p.h.replace('\\', '/')
1842 if s != p.h:
1843 p.v.h = s
1844 #@+node:ekr.20130823083943.12611: *5* ric.minimize_headlines & helper
1845 def minimize_headlines(self, p, prefix):
1846 """Create @path nodes to minimize the paths required in descendant nodes."""
1847 if prefix and not prefix.endswith('/'):
1848 prefix = prefix + '/'
1849 m = self.file_pattern.match(p.h)
1850 if m:
1851 # It's an @file node of some kind. Strip off the prefix.
1852 kind = m.group(0)
1853 path = p.h[len(kind) :].strip()
1854 stripped = self.strip_prefix(path, prefix)
1855 p.h = f"{kind} {stripped or path}"
1856 # Put the *full* @path directive in the body.
1857 if self.add_path and prefix:
1858 tail = g.os_path_dirname(stripped).rstrip('/')
1859 p.b = f"@path {prefix}{tail}\n{p.b}"
1860 else:
1861 # p.h is a path.
1862 path = p.h
1863 stripped = self.strip_prefix(path, prefix)
1864 p.h = f"@path {stripped or path}"
1865 for p in p.children():
1866 self.minimize_headlines(p, prefix + stripped)
1867 #@+node:ekr.20170404134052.1: *6* ric.strip_prefix
1868 def strip_prefix(self, path, prefix):
1869 """Strip the prefix from the path and return the result."""
1870 if path.startswith(prefix):
1871 return path[len(prefix) :]
1872 return '' # A signal.
1873 #@+node:ekr.20130823083943.12612: *5* ric.remove_empty_nodes
1874 def remove_empty_nodes(self, p):
1875 """Remove empty nodes. Not called for @auto or @edit trees."""
1876 c = self.c
1877 aList = [
1878 p2 for p2 in p.self_and_subtree()
1879 if not p2.b and not p2.hasChildren()]
1880 if aList:
1881 c.deletePositionsInList(aList) # Don't redraw.
1882 #@-others
1883#@+node:ekr.20161006071801.1: ** class TabImporter
1884class TabImporter:
1885 """
1886 A class to import a file whose outline levels are indicated by
1887 leading tabs or blanks (but not both).
1888 """
1890 def __init__(self, c, separate=True):
1891 """Ctor for the TabImporter class."""
1892 self.c = c
1893 self.root = None
1894 self.separate = separate
1895 self.stack = []
1896 #@+others
1897 #@+node:ekr.20161006071801.2: *3* tabbed.check
1898 def check(self, lines, warn=True):
1899 """Return False and warn if lines contains mixed leading tabs/blanks."""
1900 blanks, tabs = 0, 0
1901 for s in lines:
1902 lws = self.lws(s)
1903 if '\t' in lws:
1904 tabs += 1
1905 if ' ' in lws:
1906 blanks += 1
1907 if tabs and blanks:
1908 if warn:
1909 g.es_print('intermixed leading blanks and tabs.')
1910 return False
1911 return True
1912 #@+node:ekr.20161006071801.3: *3* tabbed.dump_stack
1913 def dump_stack(self):
1914 """Dump the stack, containing (level, p) tuples."""
1915 g.trace('==========')
1916 for i, data in enumerate(self.stack):
1917 level, p = data
1918 print(f"{i:2} {level} {p.h!r}")
1919 #@+node:ekr.20161006073129.1: *3* tabbed.import_files
1920 def import_files(self, files):
1921 """Import a list of tab-delimited files."""
1922 c, u = self.c, self.c.undoer
1923 if files:
1924 p = None
1925 for fn in files:
1926 try:
1927 g.setGlobalOpenDir(fn)
1928 s = open(fn).read()
1929 s = s.replace('\r', '')
1930 except Exception:
1931 continue
1932 if s.strip() and self.check(g.splitLines(s)):
1933 undoData = u.beforeInsertNode(c.p)
1934 last = c.lastTopLevel()
1935 self.root = p = last.insertAfter()
1936 self.scan(s)
1937 p.h = g.shortFileName(fn)
1938 p.contract()
1939 p.setDirty()
1940 u.afterInsertNode(p, 'Import Tabbed File', undoData)
1941 if p:
1942 c.setChanged()
1943 c.redraw(p)
1944 #@+node:ekr.20161006071801.4: *3* tabbed.lws
1945 def lws(self, s):
1946 """Return the length of the leading whitespace of s."""
1947 for i, ch in enumerate(s):
1948 if ch not in ' \t':
1949 return s[:i]
1950 return s
1951 #@+node:ekr.20161006072958.1: *3* tabbed.prompt_for_files
1952 def prompt_for_files(self):
1953 """Prompt for a list of FreeMind (.mm.html) files and import them."""
1954 c = self.c
1955 types = [
1956 ("All files", "*"),
1957 ]
1958 names = g.app.gui.runOpenFileDialog(c,
1959 title="Import Tabbed File",
1960 filetypes=types,
1961 defaultextension=".html",
1962 multiple=True)
1963 c.bringToFront()
1964 if names:
1965 g.chdir(names[0])
1966 self.import_files(names)
1967 #@+node:ekr.20161006071801.5: *3* tabbed.scan
1968 def scan(self, s1, fn=None, root=None):
1969 """Create the outline corresponding to s1."""
1970 c = self.c
1971 # Self.root can be None if we are called from a script or unit test.
1972 if not self.root:
1973 last = root if root else c.lastTopLevel() # For unit testing.
1974 self.root = last.insertAfter()
1975 if fn:
1976 self.root.h = fn
1977 lines = g.splitLines(s1)
1978 self.stack = []
1979 # Redo the checks in case we are called from a script.
1980 if s1.strip() and self.check(lines):
1981 for s in lines:
1982 if s.strip() or not self.separate:
1983 self.scan_helper(s)
1984 return self.root
1985 #@+node:ekr.20161006071801.6: *3* tabbed.scan_helper
1986 def scan_helper(self, s):
1987 """Update the stack as necessary and return level."""
1988 root, separate, stack = self.root, self.separate, self.stack
1989 if stack:
1990 level, parent = stack[-1]
1991 else:
1992 level, parent = 0, None
1993 lws = len(self.lws(s))
1994 h = s.strip()
1995 if lws == level:
1996 if separate or not parent:
1997 # Replace the top of the stack with a new entry.
1998 if stack:
1999 stack.pop()
2000 grand_parent = stack[-1][1] if stack else root
2001 parent = grand_parent.insertAsLastChild() # lws == level
2002 parent.h = h
2003 stack.append((level, parent),)
2004 elif not parent.h:
2005 parent.h = h
2006 elif lws > level:
2007 # Create a new parent.
2008 level = lws
2009 parent = parent.insertAsLastChild()
2010 parent.h = h
2011 stack.append((level, parent),)
2012 else:
2013 # Find the previous parent.
2014 while stack:
2015 level2, parent2 = stack.pop()
2016 if level2 == lws:
2017 grand_parent = stack[-1][1] if stack else root
2018 parent = grand_parent.insertAsLastChild() # lws < level
2019 parent.h = h
2020 level = lws
2021 stack.append((level, parent),)
2022 break
2023 else:
2024 level = 0
2025 parent = root.insertAsLastChild()
2026 parent.h = h
2027 stack = [(0, parent),]
2028 assert parent and parent == stack[-1][1]
2029 # An important invariant.
2030 assert level == stack[-1][0], (level, stack[-1][0])
2031 if not separate:
2032 parent.b = parent.b + self.undent(level, s)
2033 return level
2034 #@+node:ekr.20161006071801.7: *3* tabbed.undent
2035 def undent(self, level, s):
2036 """Unindent all lines of p.b by level."""
2037 if level <= 0:
2038 return s
2039 if s.strip():
2040 lines = g.splitLines(s)
2041 ch = lines[0][0]
2042 assert ch in ' \t', repr(ch)
2043 # Check that all lines start with the proper lws.
2044 lws = ch * level
2045 for s in lines:
2046 if not s.startswith(lws):
2047 g.trace(f"bad indentation: {s!r}")
2048 return s
2049 return ''.join([z[len(lws) :] for z in lines])
2050 return ''
2051 #@-others
2052#@+node:ekr.20200310060123.1: ** class ToDoImporter
2053class ToDoImporter:
2055 def __init__(self, c):
2056 self.c = c
2058 #@+others
2059 #@+node:ekr.20200310103606.1: *3* todo_i.get_tasks_from_file
2060 def get_tasks_from_file(self, path):
2061 """Return the tasks from the given path."""
2062 tag = 'import-todo-text-files'
2063 if not os.path.exists(path):
2064 print(f"{tag}: file not found: {path}")
2065 return []
2066 try:
2067 with open(path, 'r') as f:
2068 contents = f.read()
2069 tasks = self.parse_file_contents(contents)
2070 return tasks
2071 except Exception:
2072 print(f"unexpected exception in {tag}")
2073 g.es_exception()
2074 return []
2075 #@+node:ekr.20200310101028.1: *3* todo_i.import_files
2076 def import_files(self, files):
2077 """
2078 Import all todo.txt files in the given list of file names.
2080 Return a dict: keys are full paths, values are lists of ToDoTasks"
2081 """
2082 d, tag = {}, 'import-todo-text-files'
2083 for path in files:
2084 try:
2085 with open(path, 'r') as f:
2086 contents = f.read()
2087 tasks = self.parse_file_contents(contents)
2088 d[path] = tasks
2089 except Exception:
2090 print(f"unexpected exception in {tag}")
2091 g.es_exception()
2092 return d
2093 #@+node:ekr.20200310062758.1: *3* todo_i.parse_file_contents
2094 # Patterns...
2095 mark_s = r'([x]\ )'
2096 priority_s = r'(\([A-Z]\)\ )'
2097 date_s = r'([0-9]{4}-[0-9]{2}-[0-9]{2}\ )'
2098 task_s = r'\s*(.+)'
2099 line_s = fr"^{mark_s}?{priority_s}?{date_s}?{date_s}?{task_s}$"
2100 line_pat = re.compile(line_s)
2102 def parse_file_contents(self, s):
2103 """
2104 Parse the contents of a file.
2105 Return a list of ToDoTask objects.
2106 """
2107 trace = False
2108 tasks = []
2109 for line in g.splitLines(s):
2110 if not line.strip():
2111 continue
2112 if trace:
2113 print(f"task: {line.rstrip()!s}")
2114 m = self.line_pat.match(line)
2115 if not m:
2116 print(f"invalid task: {line.rstrip()!s}")
2117 continue
2118 # Groups 1, 2 and 5 are context independent.
2119 completed = m.group(1)
2120 priority = m.group(2)
2121 task_s = m.group(5)
2122 if not task_s:
2123 print(f"invalid task: {line.rstrip()!s}")
2124 continue
2125 # Groups 3 and 4 are context dependent.
2126 if m.group(3) and m.group(4):
2127 complete_date = m.group(3)
2128 start_date = m.group(4)
2129 elif completed:
2130 complete_date = m.group(3)
2131 start_date = ''
2132 else:
2133 start_date = m.group(3) or ''
2134 complete_date = ''
2135 if completed and not complete_date:
2136 print(f"no completion date: {line.rstrip()!s}")
2137 tasks.append(ToDoTask(
2138 bool(completed), priority, start_date, complete_date, task_s))
2139 return tasks
2140 #@+node:ekr.20200310100919.1: *3* todo_i.prompt_for_files
2141 def prompt_for_files(self):
2142 """
2143 Prompt for a list of todo.text files and import them.
2145 Return a python dict. Keys are full paths; values are lists of ToDoTask objects.
2146 """
2147 c = self.c
2148 types = [
2149 ("Text files", "*.txt"),
2150 ("All files", "*"),
2151 ]
2152 names = g.app.gui.runOpenFileDialog(c,
2153 title="Import todo.txt File",
2154 filetypes=types,
2155 defaultextension=".txt",
2156 multiple=True,
2157 )
2158 c.bringToFront()
2159 if not names:
2160 return {}
2161 g.chdir(names[0])
2162 d = self.import_files(names)
2163 for key in sorted(d):
2164 tasks = d.get(key)
2165 print(f"tasks in {g.shortFileName(key)}...\n")
2166 for task in tasks:
2167 print(f" {task}")
2168 return d
2169 #@-others
2170#@+node:ekr.20200310063208.1: ** class ToDoTask
2171class ToDoTask:
2172 """A class representing the components of a task line."""
2174 def __init__(self, completed, priority, start_date, complete_date, task_s):
2175 self.completed = completed
2176 self.priority = priority and priority[1] or ''
2177 self.start_date = start_date and start_date.rstrip() or ''
2178 self.complete_date = complete_date and complete_date.rstrip() or ''
2179 self.task_s = task_s.strip()
2180 # Parse tags into separate dictionaries.
2181 self.projects = []
2182 self.contexts = []
2183 self.key_vals = []
2184 self.parse_task()
2186 #@+others
2187 #@+node:ekr.20200310075514.1: *3* task.__repr__ & __str__
2188 def __repr__(self):
2189 start_s = self.start_date if self.start_date else ''
2190 end_s = self.complete_date if self.complete_date else ''
2191 mark_s = '[X]' if self.completed else '[ ]'
2192 result = [
2193 f"Task: "
2194 f"{mark_s} "
2195 f"{self.priority:1} "
2196 f"start: {start_s:10} "
2197 f"end: {end_s:10} "
2198 f"{self.task_s}"
2199 ]
2200 for ivar in ('contexts', 'projects', 'key_vals'):
2201 aList = getattr(self, ivar, None)
2202 if aList:
2203 result.append(f"{' '*13}{ivar}: {aList}")
2204 return '\n'.join(result)
2206 __str__ = __repr__
2207 #@+node:ekr.20200310063138.1: *3* task.parse_task
2208 # Patterns...
2209 project_pat = re.compile(r'(\+\S+)')
2210 context_pat = re.compile(r'(@\S+)')
2211 key_val_pat = re.compile(r'((\S+):(\S+))') # Might be a false match.
2213 def parse_task(self):
2215 trace = False and not g.unitTesting
2216 s = self.task_s
2217 table = (
2218 ('context', self.context_pat, self.contexts),
2219 ('project', self.project_pat, self.projects),
2220 ('key:val', self.key_val_pat, self.key_vals),
2221 )
2222 for kind, pat, aList in table:
2223 for m in re.finditer(pat, s):
2224 pat_s = repr(pat).replace("re.compile('", "").replace("')", "")
2225 pat_s = pat_s.replace(r'\\', '\\')
2226 # Check for false key:val match:
2227 if pat == self.key_val_pat:
2228 key, value = m.group(2), m.group(3)
2229 if ':' in key or ':' in value:
2230 break
2231 tag = m.group(1)
2232 # Add the tag.
2233 if tag in aList:
2234 if trace:
2235 g.trace('Duplicate tag:', tag)
2236 else:
2237 if trace:
2238 g.trace(f"Add {kind} tag: {tag!s}")
2239 aList.append(tag)
2240 # Remove the tag from the task.
2241 s = re.sub(pat, "", s)
2242 if s != self.task_s:
2243 self.task_s = s.strip()
2244 #@-others
2245#@+node:ekr.20141210051628.26: ** class ZimImportController
2246class ZimImportController:
2247 """
2248 A class to import Zim folders and files: http://zim-wiki.org/
2249 First use Zim to export your project to rst files.
2251 Original script by Davy Cottet.
2253 User options:
2254 @int rst_level = 0
2255 @string rst_type
2256 @string zim_node_name
2257 @string path_to_zim
2259 """
2260 #@+others
2261 #@+node:ekr.20141210051628.31: *3* zic.__init__ & zic.reloadSettings
2262 def __init__(self, c):
2263 """Ctor for ZimImportController class."""
2264 self.c = c
2265 self.pathToZim = c.config.getString('path-to-zim')
2266 self.rstLevel = c.config.getInt('zim-rst-level') or 0
2267 self.rstType = c.config.getString('zim-rst-type') or 'rst'
2268 self.zimNodeName = c.config.getString('zim-node-name') or 'Imported Zim Tree'
2269 #@+node:ekr.20141210051628.28: *3* zic.parseZimIndex
2270 def parseZimIndex(self):
2271 """
2272 Parse Zim wiki index.rst and return a list of tuples (level, name, path) or None.
2273 """
2274 # c = self.c
2275 pathToZim = g.os_path_abspath(self.pathToZim)
2276 pathToIndex = g.os_path_join(pathToZim, 'index.rst')
2277 if not g.os_path_exists(pathToIndex):
2278 g.es(f"not found: {pathToIndex}", color='red')
2279 return None
2280 index = open(pathToIndex).read()
2281 parse = re.findall(r'(\t*)-\s`(.+)\s<(.+)>`_', index)
2282 if not parse:
2283 g.es(f"invalid index: {pathToIndex}", color='red')
2284 return None
2285 results = []
2286 for result in parse:
2287 level = len(result[0])
2288 name = result[1].decode('utf-8')
2289 unquote = urllib.parse.unquote
2290 # mypy: error: "str" has no attribute "decode"; maybe "encode"? [attr-defined]
2291 path = [g.os_path_abspath(g.os_path_join(
2292 pathToZim, unquote(result[2]).decode('utf-8')))] # type:ignore
2293 results.append((level, name, path))
2294 return results
2295 #@+node:ekr.20141210051628.29: *3* zic.rstToLastChild
2296 def rstToLastChild(self, p, name, rst):
2297 """Import an rst file as a last child of pos node with the specified name"""
2298 c = self.c
2299 c.importCommands.importFilesCommand(
2300 files=rst,
2301 parent=p,
2302 treeType='@rst',
2303 )
2304 rstNode = p.getLastChild()
2305 rstNode.h = name
2306 return rstNode
2307 #@+node:davy.20141212140940.1: *3* zic.clean
2308 def clean(self, zimNode, rstType):
2309 """Clean useless nodes"""
2310 warning = 'Warning: this node is ignored when writing this file'
2311 for p in zimNode.subtree_iter():
2312 # looking for useless bodies
2313 if p.hasFirstChild() and warning in p.b:
2314 child = p.getFirstChild()
2315 fmt = "@rst-no-head %s declarations"
2316 table = (
2317 fmt % p.h.replace(' ', '_'),
2318 fmt % p.h.replace(rstType, '').strip().replace(' ', '_'),
2319 )
2320 # Replace content with @rest-no-head first child (without title head) and delete it
2321 if child.h in table:
2322 p.b = '\n'.join(child.b.split('\n')[3:])
2323 child.doDelete()
2324 # Replace content of empty body parent node with first child with same name
2325 elif p.h == child.h or (f"{rstType} {child.h}" == p.h):
2326 if not child.hasFirstChild():
2327 p.b = child.b
2328 child.doDelete()
2329 elif not child.hasNext():
2330 p.b = child.b
2331 child.copyTreeFromSelfTo(p)
2332 child.doDelete()
2333 else:
2334 child.h = 'Introduction'
2335 elif p.hasFirstChild(
2336 ) and p.h.startswith("@rst-no-head") and not p.b.strip():
2337 child = p.getFirstChild()
2338 p_no_head = p.h.replace("@rst-no-head", "").strip()
2339 # Replace empty @rst-no-head by its same named chidren
2340 if child.h.strip() == p_no_head and not child.hasFirstChild():
2341 p.h = p_no_head
2342 p.b = child.b
2343 child.doDelete()
2344 elif p.h.startswith("@rst-no-head"):
2345 lines = p.b.split('\n')
2346 p.h = lines[1]
2347 p.b = '\n'.join(lines[3:])
2348 #@+node:ekr.20141210051628.30: *3* zic.run
2349 def run(self):
2350 """Create the zim node as the last top-level node."""
2351 c = self.c
2352 # Make sure a path is given.
2353 if not self.pathToZim:
2354 g.es('Missing setting: @string path_to_zim', color='red')
2355 return
2356 root = c.rootPosition()
2357 while root.hasNext():
2358 root.moveToNext()
2359 zimNode = root.insertAfter()
2360 zimNode.h = self.zimNodeName
2361 # Parse the index file
2362 files = self.parseZimIndex()
2363 if files:
2364 # Do the import
2365 rstNodes = {'0': zimNode,}
2366 for level, name, rst in files:
2367 if level == self.rstLevel:
2368 name = f"{self.rstType} {name}"
2369 rstNodes[
2370 str(
2371 level + 1)] = self.rstToLastChild(rstNodes[str(level)], name, rst)
2372 # Clean nodes
2373 g.es('Start cleaning process. Please wait...', color='blue')
2374 self.clean(zimNode, self.rstType)
2375 g.es('Done', color='blue')
2376 # Select zimNode
2377 c.selectPosition(zimNode)
2378 c.redraw()
2379 #@-others
2380#@+node:ekr.20200424152850.1: ** class LegacyExternalFileImporter
2381class LegacyExternalFileImporter:
2382 """
2383 A class to import external files written by versions of Leo earlier
2384 than 5.0.
2385 """
2386 # Sentinels to ignore, without the leading comment delim.
2387 ignore = ('@+at', '@-at', '@+leo', '@-leo', '@nonl', '@nl', '@-others')
2389 def __init__(self, c):
2390 self.c = c
2392 #@+others
2393 #@+node:ekr.20200424093946.1: *3* class Node
2394 class Node:
2396 def __init__(self, h, level):
2397 """Hold node data."""
2398 self.h = h.strip()
2399 self.level = level
2400 self.lines = []
2401 #@+node:ekr.20200424092652.1: *3* legacy.add
2402 def add(self, line, stack):
2403 """Add a line to the present node."""
2404 if stack:
2405 node = stack[-1]
2406 node.lines.append(line)
2407 else:
2408 print('orphan line: ', repr(line))
2409 #@+node:ekr.20200424160847.1: *3* legacy.compute_delim1
2410 def compute_delim1(self, path):
2411 """Return the opening comment delim for the given file."""
2412 junk, ext = os.path.splitext(path)
2413 if not ext:
2414 return None
2415 language = g.app.extension_dict.get(ext[1:])
2416 if not language:
2417 return None
2418 delim1, delim2, delim3 = g.set_delims_from_language(language)
2419 g.trace(language, delim1 or delim2)
2420 return delim1 or delim2
2421 #@+node:ekr.20200424153139.1: *3* legacy.import_file
2422 def import_file(self, path):
2423 """Import one legacy external file."""
2424 c = self.c
2425 root_h = g.shortFileName(path)
2426 delim1 = self.compute_delim1(path)
2427 if not delim1:
2428 g.es_print('unknown file extension:', color='red')
2429 g.es_print(path)
2430 return
2431 # Read the file into s.
2432 with open(path, 'r') as f:
2433 s = f.read()
2434 # Do nothing if the file is a newer external file.
2435 if delim1 + '@+leo-ver=4' not in s:
2436 g.es_print('not a legacy external file:', color='red')
2437 g.es_print(path)
2438 return
2439 # Compute the local ignore list for this file.
2440 ignore = tuple(delim1 + z for z in self.ignore)
2441 # Handle each line of the file.
2442 nodes: List[Any] = [] # An list of Nodes, in file order.
2443 stack: List[Any] = [] # A stack of Nodes.
2444 for line in g.splitLines(s):
2445 s = line.lstrip()
2446 lws = line[: len(line) - len(line.lstrip())]
2447 if s.startswith(delim1 + '@@'):
2448 self.add(lws + s[2:], stack)
2449 elif s.startswith(ignore):
2450 # Ignore these. Use comments instead of @doc bodies.
2451 pass
2452 elif (
2453 s.startswith(delim1 + '@+others') or
2454 s.startswith(delim1 + '@' + lws + '@+others')
2455 ):
2456 self.add(lws + '@others\n', stack)
2457 elif s.startswith(delim1 + '@<<'):
2458 n = len(delim1 + '@<<')
2459 self.add(lws + '<<' + s[n:].rstrip() + '\n', stack)
2460 elif s.startswith(delim1 + '@+node:'):
2461 # Compute the headline.
2462 if stack:
2463 h = s[8:]
2464 i = h.find(':')
2465 h = h[i + 1 :] if ':' in h else h
2466 else:
2467 h = root_h
2468 # Create a node and push it.
2469 node = self.Node(h, len(stack))
2470 nodes.append(node)
2471 stack.append(node)
2472 elif s.startswith(delim1 + '@-node'):
2473 # End the node.
2474 stack.pop()
2475 elif s.startswith(delim1 + '@'):
2476 print('oops:', repr(s))
2477 else:
2478 self.add(line, stack)
2479 if stack:
2480 print('Unbalanced node sentinels')
2481 # Generate nodes.
2482 last = c.lastTopLevel()
2483 root = last.insertAfter()
2484 root.h = f"imported file: {root_h}"
2485 stack = [root]
2486 for node in nodes:
2487 b = textwrap.dedent(''.join(node.lines))
2488 level = node.level
2489 if level == 0:
2490 root.h = root_h
2491 root.b = b
2492 else:
2493 parent = stack[level - 1]
2494 p = parent.insertAsLastChild()
2495 p.b = b
2496 p.h = node.h
2497 # Good for debugging.
2498 # p.h = f"{level} {node.h}"
2499 stack = stack[:level] + [p]
2500 c.selectPosition(root)
2501 root.expand() # c.expandAllSubheads()
2502 c.redraw()
2503 #@+node:ekr.20200424154553.1: *3* legacy.import_files
2504 def import_files(self, paths):
2505 """Import zero or more files."""
2506 for path in paths:
2507 if os.path.exists(path):
2508 self.import_file(path)
2509 else:
2510 g.es_print(f"not found: {path!r}")
2511 #@+node:ekr.20200424154416.1: *3* legacy.prompt_for_files
2512 def prompt_for_files(self):
2513 """Prompt for a list of legacy external .py files and import them."""
2514 c = self.c
2515 types = [
2516 ("Legacy external files", "*.py"),
2517 ("All files", "*"),
2518 ]
2519 paths = g.app.gui.runOpenFileDialog(c,
2520 title="Import Legacy External Files",
2521 filetypes=types,
2522 defaultextension=".py",
2523 multiple=True)
2524 c.bringToFront()
2525 if paths:
2526 g.chdir(paths[0])
2527 self.import_files(paths)
2528 #@-others
2529#@+node:ekr.20101103093942.5938: ** Commands (leoImport)
2530#@+node:ekr.20160504050255.1: *3* @g.command(import-free-mind-files)
2531@g.command('import-free-mind-files')
2532def import_free_mind_files(event):
2533 """Prompt for free-mind files and import them."""
2534 c = event.get('c')
2535 if c:
2536 FreeMindImporter(c).prompt_for_files()
2538#@+node:ekr.20200424154303.1: *3* @g.command(import-legacy-external-file)
2539@g.command('import-legacy-external-files')
2540def import_legacy_external_files(event):
2541 """Prompt for legacy external files and import them."""
2542 c = event.get('c')
2543 if c:
2544 LegacyExternalFileImporter(c).prompt_for_files()
2545#@+node:ekr.20160504050325.1: *3* @g.command(import-mind-map-files
2546@g.command('import-mind-jet-files')
2547def import_mind_jet_files(event):
2548 """Prompt for mind-jet files and import them."""
2549 c = event.get('c')
2550 if c:
2551 MindMapImporter(c).prompt_for_files()
2552#@+node:ekr.20161006100854.1: *3* @g.command(import-MORE-files)
2553@g.command('import-MORE-files')
2554def import_MORE_files_command(event):
2555 """Prompt for MORE files and import them."""
2556 c = event.get('c')
2557 if c:
2558 MORE_Importer(c).prompt_for_files()
2559#@+node:ekr.20161006072227.1: *3* @g.command(import-tabbed-files)
2560@g.command('import-tabbed-files')
2561def import_tabbed_files_command(event):
2562 """Prompt for tabbed files and import them."""
2563 c = event.get('c')
2564 if c:
2565 TabImporter(c).prompt_for_files()
2566#@+node:ekr.20200310095703.1: *3* @g.command(import-todo-text-files)
2567@g.command('import-todo-text-files')
2568def import_todo_text_files(event):
2569 """Prompt for free-mind files and import them."""
2570 c = event.get('c')
2571 if c:
2572 ToDoImporter(c).prompt_for_files()
2573#@+node:ekr.20141210051628.33: *3* @g.command(import-zim-folder)
2574@g.command('import-zim-folder')
2575def import_zim_command(event):
2576 """
2577 Import a zim folder, http://zim-wiki.org/, as the last top-level node of the outline.
2579 First use Zim to export your project to rst files.
2581 This command requires the following Leo settings::
2583 @int rst_level = 0
2584 @string rst_type
2585 @string zim_node_name
2586 @string path_to_zim
2587 """
2588 c = event.get('c')
2589 if c:
2590 ZimImportController(c).run()
2591#@+node:ekr.20120429125741.10057: *3* @g.command(parse-body)
2592@g.command('parse-body')
2593def parse_body_command(event):
2594 """The parse-body command."""
2595 c = event.get('c')
2596 if c and c.p:
2597 c.importCommands.parse_body(c.p)
2598#@-others
2599#@@language python
2600#@@tabwidth -4
2601#@@pagewidth 70
2602#@@encoding utf-8
2603#@-leo