Coverage for C:\Repos\leo-editor\leo\core\leoAtFile.py: 99%
996 statements
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20150323150718.1: * @file leoAtFile.py
4#@@first
5"""Classes to read and write @file nodes."""
6#@+<< imports >>
7#@+node:ekr.20041005105605.2: ** << imports >> (leoAtFile.py)
8import io
9import os
10import re
11import sys
12import tabnanny
13import time
14import tokenize
15from typing import List
16from leo.core import leoGlobals as g
17from leo.core import leoNodes
18#@-<< imports >>
19#@+others
20#@+node:ekr.20150509194251.1: ** cmd (decorator)
21def cmd(name): # pragma: no cover
22 """Command decorator for the AtFileCommands class."""
23 return g.new_cmd_decorator(name, ['c', 'atFileCommands',])
24#@+node:ekr.20160514120655.1: ** class AtFile
25class AtFile:
26 """A class implementing the atFile subcommander."""
27 #@+<< define class constants >>
28 #@+node:ekr.20131224053735.16380: *3* << define class constants >>
29 #@@nobeautify
31 # directives...
32 noDirective = 1 # not an at-directive.
33 allDirective = 2 # at-all (4.2)
34 docDirective = 3 # @doc.
35 atDirective = 4 # @<space> or @<newline>
36 codeDirective = 5 # @code
37 cDirective = 6 # @c<space> or @c<newline>
38 othersDirective = 7 # at-others
39 miscDirective = 8 # All other directives
40 startVerbatim = 9 # @verbatim Not a real directive. Used to issue warnings.
41 #@-<< define class constants >>
42 #@+others
43 #@+node:ekr.20041005105605.7: *3* at.Birth & init
44 #@+node:ekr.20041005105605.8: *4* at.ctor & helpers
45 # Note: g.getScript also call the at.__init__ and at.finishCreate().
47 def __init__(self, c):
48 """ctor for atFile class."""
49 # **Warning**: all these ivars must **also** be inited in initCommonIvars.
50 self.c = c
51 self.encoding = 'utf-8' # 2014/08/13
52 self.fileCommands = c.fileCommands
53 self.errors = 0 # Make sure at.error() works even when not inited.
54 # #2276: allow different section delims.
55 self.section_delim1 = '<<'
56 self.section_delim2 = '>>'
57 # **Only** at.writeAll manages these flags.
58 self.unchangedFiles = 0
59 # promptForDangerousWrite sets cancelFlag and yesToAll only if canCancelFlag is True.
60 self.canCancelFlag = False
61 self.cancelFlag = False
62 self.yesToAll = False
63 # User options: set in reloadSettings.
64 self.checkPythonCodeOnWrite = False
65 self.runPyFlakesOnWrite = False
66 self.underindentEscapeString = '\\-'
67 self.reloadSettings()
68 #@+node:ekr.20171113152939.1: *5* at.reloadSettings
69 def reloadSettings(self):
70 """AtFile.reloadSettings"""
71 c = self.c
72 self.checkPythonCodeOnWrite = c.config.getBool(
73 'check-python-code-on-write', default=True)
74 self.runPyFlakesOnWrite = c.config.getBool(
75 'run-pyflakes-on-write', default=False)
76 self.underindentEscapeString = c.config.getString(
77 'underindent-escape-string') or '\\-'
78 #@+node:ekr.20041005105605.10: *4* at.initCommonIvars
79 def initCommonIvars(self):
80 """
81 Init ivars common to both reading and writing.
83 The defaults set here may be changed later.
84 """
85 at = self
86 c = at.c
87 at.at_auto_encoding = c.config.default_at_auto_file_encoding
88 at.encoding = c.config.default_derived_file_encoding
89 at.endSentinelComment = ""
90 at.errors = 0
91 at.inCode = True
92 at.indent = 0 # The unit of indentation is spaces, not tabs.
93 at.language = None
94 at.output_newline = g.getOutputNewline(c=c)
95 at.page_width = None
96 at.root = None # The root (a position) of tree being read or written.
97 at.startSentinelComment = ""
98 at.startSentinelComment = ""
99 at.tab_width = c.tab_width or -4
100 at.writing_to_shadow_directory = False
101 #@+node:ekr.20041005105605.13: *4* at.initReadIvars
102 def initReadIvars(self, root, fileName):
103 at = self
104 at.initCommonIvars()
105 at.bom_encoding = None # The encoding implied by any BOM (set by g.stripBOM)
106 at.cloneSibCount = 0 # n > 1: Make sure n cloned sibs exists at next @+node sentinel
107 at.correctedLines = 0 # For perfect import.
108 at.docOut = [] # The doc part being accumulated.
109 at.done = False # True when @-leo seen.
110 at.fromString = False
111 at.importRootSeen = False
112 at.indentStack = []
113 at.lastLines = [] # The lines after @-leo
114 at.leadingWs = ""
115 at.lineNumber = 0 # New in Leo 4.4.8.
116 at.out = None
117 at.outStack = []
118 at.read_i = 0
119 at.read_lines = []
120 at.readVersion = '' # "5" for new-style thin files.
121 at.readVersion5 = False # Synonym for at.readVersion >= '5'
122 at.root = root
123 at.rootSeen = False
124 at.targetFileName = fileName # For at.writeError only.
125 at.v = None
126 at.vStack = [] # Stack of at.v values.
127 at.thinChildIndexStack = [] # number of siblings at this level.
128 at.thinNodeStack = [] # Entries are vnodes.
129 at.updateWarningGiven = False
130 #@+node:ekr.20041005105605.15: *4* at.initWriteIvars
131 def initWriteIvars(self, root):
132 """
133 Compute default values of all write-related ivars.
134 Return the finalized name of the output file.
135 """
136 at, c = self, self.c
137 if not c and c.config:
138 return None # pragma: no cover
139 make_dirs = c.config.create_nonexistent_directories
140 assert root
141 self.initCommonIvars()
142 assert at.checkPythonCodeOnWrite is not None
143 assert at.underindentEscapeString is not None
144 #
145 # Copy args
146 at.root = root
147 at.sentinels = True
148 #
149 # Override initCommonIvars.
150 if g.unitTesting:
151 at.output_newline = '\n'
152 #
153 # Set other ivars.
154 at.force_newlines_in_at_nosent_bodies = c.config.getBool(
155 'force-newlines-in-at-nosent-bodies')
156 # For at.putBody only.
157 at.outputList = [] # For stream output.
158 # Sets the following ivars:
159 # at.encoding
160 # at.explicitLineEnding
161 # at.language
162 # at.output_newline
163 # at.page_width
164 # at.tab_width
165 at.scanAllDirectives(root)
166 #
167 # Overrides of at.scanAllDirectives...
168 if at.language == 'python':
169 # Encoding directive overrides everything else.
170 encoding = g.getPythonEncodingFromString(root.b)
171 if encoding:
172 at.encoding = encoding
173 #
174 # Clean root.v.
175 if not at.errors and at.root:
176 at.root.v._p_changed = True
177 #
178 # #1907: Compute the file name and create directories as needed.
179 targetFileName = g.os_path_realpath(g.fullPath(c, root))
180 at.targetFileName = targetFileName # For at.writeError only.
181 #
182 # targetFileName can be empty for unit tests & @command nodes.
183 if not targetFileName: # pragma: no cover
184 targetFileName = root.h if g.unitTesting else None
185 at.targetFileName = targetFileName # For at.writeError only.
186 return targetFileName
187 #
188 # #2276: scan for section delims
189 at.scanRootForSectionDelims(root)
190 #
191 # Do nothing more if the file already exists.
192 if os.path.exists(targetFileName):
193 return targetFileName
194 #
195 # Create directories if enabled.
196 root_dir = g.os_path_dirname(targetFileName)
197 if make_dirs and root_dir: # pragma: no cover
198 ok = g.makeAllNonExistentDirectories(root_dir)
199 if not ok:
200 g.error(f"Error creating directories: {root_dir}")
201 return None
202 #
203 # Return the target file name, regardless of future problems.
204 return targetFileName
205 #@+node:ekr.20041005105605.17: *3* at.Reading
206 #@+node:ekr.20041005105605.18: *4* at.Reading (top level)
207 #@+node:ekr.20070919133659: *5* at.checkExternalFile
208 @cmd('check-external-file')
209 def checkExternalFile(self, event=None): # pragma: no cover
210 """Make sure an external file written by Leo may be read properly."""
211 c, p = self.c, self.c.p
212 if not p.isAtFileNode() and not p.isAtThinFileNode():
213 g.red('Please select an @thin or @file node')
214 return
215 fn = g.fullPath(c, p) # #1910.
216 if not g.os_path_exists(fn):
217 g.red(f"file not found: {fn}")
218 return
219 s, e = g.readFileIntoString(fn)
220 if s is None:
221 g.red(f"empty file: {fn}")
222 return
223 #
224 # Create a dummy, unconnected, VNode as the root.
225 root_v = leoNodes.VNode(context=c)
226 root = leoNodes.Position(root_v)
227 FastAtRead(c, gnx2vnode={}).read_into_root(s, fn, root)
228 #@+node:ekr.20041005105605.19: *5* at.openFileForReading & helper
229 def openFileForReading(self, fromString=False):
230 """
231 Open the file given by at.root.
232 This will be the private file for @shadow nodes.
233 """
234 at, c = self, self.c
235 is_at_shadow = self.root.isAtShadowFileNode()
236 if fromString: # pragma: no cover
237 if is_at_shadow: # pragma: no cover
238 return at.error(
239 'can not call at.read from string for @shadow files')
240 at.initReadLine(fromString)
241 return None, None
242 #
243 # Not from a string. Carefully read the file.
244 # Returns full path, including file name.
245 fn = g.fullPath(c, at.root)
246 # Remember the full path to this node.
247 at.setPathUa(at.root, fn)
248 if is_at_shadow: # pragma: no cover
249 fn = at.openAtShadowFileForReading(fn)
250 if not fn:
251 return None, None
252 assert fn
253 try:
254 # Sets at.encoding, regularizes whitespace and calls at.initReadLines.
255 s = at.readFileToUnicode(fn)
256 # #1466.
257 if s is None: # pragma: no cover
258 # The error has been given.
259 at._file_bytes = g.toEncodedString('')
260 return None, None
261 at.warnOnReadOnlyFile(fn)
262 except Exception: # pragma: no cover
263 at.error(f"unexpected exception opening: '@file {fn}'")
264 at._file_bytes = g.toEncodedString('')
265 fn, s = None, None
266 return fn, s
267 #@+node:ekr.20150204165040.4: *6* at.openAtShadowFileForReading
268 def openAtShadowFileForReading(self, fn): # pragma: no cover
269 """Open an @shadow for reading and return shadow_fn."""
270 at = self
271 x = at.c.shadowController
272 # readOneAtShadowNode should already have checked these.
273 shadow_fn = x.shadowPathName(fn)
274 shadow_exists = (g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn))
275 if not shadow_exists:
276 g.trace('can not happen: no private file',
277 shadow_fn, g.callers())
278 at.error(f"can not happen: private file does not exist: {shadow_fn}")
279 return None
280 # This method is the gateway to the shadow algorithm.
281 x.updatePublicAndPrivateFiles(at.root, fn, shadow_fn)
282 return shadow_fn
283 #@+node:ekr.20041005105605.21: *5* at.read & helpers
284 def read(self, root, fromString=None):
285 """Read an @thin or @file tree."""
286 at, c = self, self.c
287 fileName = g.fullPath(c, root) # #1341. #1889.
288 if not fileName: # pragma: no cover
289 at.error("Missing file name. Restoring @file tree from .leo file.")
290 return False
291 # Fix bug 760531: always mark the root as read, even if there was an error.
292 # Fix bug 889175: Remember the full fileName.
293 at.rememberReadPath(g.fullPath(c, root), root)
294 at.initReadIvars(root, fileName)
295 at.fromString = fromString
296 if at.errors:
297 return False # pragma: no cover
298 fileName, file_s = at.openFileForReading(fromString=fromString)
299 # #1798:
300 if file_s is None:
301 return False # pragma: no cover
302 #
303 # Set the time stamp.
304 if fileName:
305 c.setFileTimeStamp(fileName)
306 elif not fileName and not fromString and not file_s: # pragma: no cover
307 return False
308 root.clearVisitedInTree()
309 # Sets the following ivars:
310 # at.encoding: **changed later** by readOpenFile/at.scanHeader.
311 # at.explicitLineEnding
312 # at.language
313 # at.output_newline
314 # at.page_width
315 # at.tab_width
316 at.scanAllDirectives(root)
317 gnx2vnode = c.fileCommands.gnxDict
318 contents = fromString or file_s
319 FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
320 root.clearDirty()
321 return True
322 #@+node:ekr.20071105164407: *6* at.deleteUnvisitedNodes
323 def deleteUnvisitedNodes(self, root): # pragma: no cover
324 """
325 Delete unvisited nodes in root's subtree, not including root.
327 Before Leo 5.6: Move unvisited node to be children of the 'Resurrected
328 Nodes'.
329 """
330 at, c = self, self.c
331 # Find the unvisited nodes.
332 aList = [z for z in root.subtree() if not z.isVisited()]
333 if aList:
334 at.c.deletePositionsInList(aList)
335 c.redraw()
337 #@+node:ekr.20041005105605.26: *5* at.readAll & helpers
338 def readAll(self, root):
339 """Scan positions, looking for @<file> nodes to read."""
340 at, c = self, self.c
341 old_changed = c.changed
342 t1 = time.time()
343 c.init_error_dialogs()
344 files = at.findFilesToRead(root, all=True)
345 for p in files:
346 at.readFileAtPosition(p)
347 for p in files:
348 p.v.clearDirty()
349 if not g.unitTesting and files: # pragma: no cover
350 t2 = time.time()
351 g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")
352 c.changed = old_changed
353 c.raise_error_dialogs()
354 #@+node:ekr.20190108054317.1: *6* at.findFilesToRead
355 def findFilesToRead(self, root, all): # pragma: no cover
357 c = self.c
358 p = root.copy()
359 scanned_nodes = set()
360 files = []
361 after = None if all else p.nodeAfterTree()
362 while p and p != after:
363 data = (p.gnx, g.fullPath(c, p))
364 # skip clones referring to exactly the same paths.
365 if data in scanned_nodes:
366 p.moveToNodeAfterTree()
367 continue
368 scanned_nodes.add(data)
369 if not p.h.startswith('@'):
370 p.moveToThreadNext()
371 elif p.isAtIgnoreNode():
372 if p.isAnyAtFileNode():
373 c.ignored_at_file_nodes.append(p.h)
374 p.moveToNodeAfterTree()
375 elif (
376 p.isAtThinFileNode() or
377 p.isAtAutoNode() or
378 p.isAtEditNode() or
379 p.isAtShadowFileNode() or
380 p.isAtFileNode() or
381 p.isAtCleanNode() # 1134.
382 ):
383 files.append(p.copy())
384 p.moveToNodeAfterTree()
385 elif p.isAtAsisFileNode() or p.isAtNoSentFileNode():
386 # Note (see #1081): @asis and @nosent can *not* be updated automatically.
387 # Doing so using refresh-from-disk will delete all child nodes.
388 p.moveToNodeAfterTree()
389 else:
390 p.moveToThreadNext()
391 return files
392 #@+node:ekr.20190108054803.1: *6* at.readFileAtPosition
393 def readFileAtPosition(self, p): # pragma: no cover
394 """Read the @<file> node at p."""
395 at, c, fileName = self, self.c, p.anyAtFileNodeName()
396 if p.isAtThinFileNode() or p.isAtFileNode():
397 at.read(p)
398 elif p.isAtAutoNode():
399 at.readOneAtAutoNode(p)
400 elif p.isAtEditNode():
401 at.readOneAtEditNode(fileName, p)
402 elif p.isAtShadowFileNode():
403 at.readOneAtShadowNode(fileName, p)
404 elif p.isAtAsisFileNode() or p.isAtNoSentFileNode():
405 at.rememberReadPath(g.fullPath(c, p), p)
406 elif p.isAtCleanNode():
407 at.readOneAtCleanNode(p)
408 #@+node:ekr.20220121052056.1: *5* at.readAllSelected
409 def readAllSelected(self, root): # pragma: no cover
410 """Read all @<file> nodes in root's tree."""
411 at, c = self, self.c
412 old_changed = c.changed
413 t1 = time.time()
414 c.init_error_dialogs()
415 files = at.findFilesToRead(root, all=False)
416 for p in files:
417 at.readFileAtPosition(p)
418 for p in files:
419 p.v.clearDirty()
420 if not g.unitTesting: # pragma: no cover
421 if files:
422 t2 = time.time()
423 g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")
424 else:
425 g.es("no @<file> nodes in the selected tree")
426 c.changed = old_changed
427 c.raise_error_dialogs()
428 #@+node:ekr.20080801071227.7: *5* at.readAtShadowNodes
429 def readAtShadowNodes(self, p): # pragma: no cover
430 """Read all @shadow nodes in the p's tree."""
431 at = self
432 after = p.nodeAfterTree()
433 p = p.copy() # Don't change p in the caller.
434 while p and p != after: # Don't use iterator.
435 if p.isAtShadowFileNode():
436 fileName = p.atShadowFileNodeName()
437 at.readOneAtShadowNode(fileName, p)
438 p.moveToNodeAfterTree()
439 else:
440 p.moveToThreadNext()
441 #@+node:ekr.20070909100252: *5* at.readOneAtAutoNode
442 def readOneAtAutoNode(self, p): # pragma: no cover
443 """Read an @auto file into p. Return the *new* position."""
444 at, c, ic = self, self.c, self.c.importCommands
445 fileName = g.fullPath(c, p) # #1521, #1341, #1914.
446 if not g.os_path_exists(fileName):
447 g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
448 return p
449 # Remember that we have seen the @auto node.
450 # #889175: Remember the full fileName.
451 at.rememberReadPath(fileName, p)
452 old_p = p.copy()
453 try:
454 at.scanAllDirectives(p)
455 p.v.b = '' # Required for @auto API checks.
456 p.v._deleteAllChildren()
457 p = ic.createOutline(parent=p.copy())
458 # Do *not* call c.selectPosition(p) here.
459 # That would improperly expand nodes.
460 except Exception:
461 p = old_p
462 ic.errors += 1
463 g.es_print('Unexpected exception importing', fileName)
464 g.es_exception()
465 if ic.errors:
466 g.error(f"errors inhibited read @auto {fileName}")
467 elif c.persistenceController:
468 c.persistenceController.update_after_read_foreign_file(p)
469 # Finish.
470 if ic.errors or not g.os_path_exists(fileName):
471 p.clearDirty()
472 else:
473 g.doHook('after-auto', c=c, p=p)
474 return p # For #451: return p.
475 #@+node:ekr.20090225080846.3: *5* at.readOneAtEditNode
476 def readOneAtEditNode(self, fn, p): # pragma: no cover
477 at = self
478 c = at.c
479 ic = c.importCommands
480 # #1521
481 fn = g.fullPath(c, p)
482 junk, ext = g.os_path_splitext(fn)
483 # Fix bug 889175: Remember the full fileName.
484 at.rememberReadPath(fn, p)
485 # if not g.unitTesting: g.es("reading: @edit %s" % (g.shortFileName(fn)))
486 s, e = g.readFileIntoString(fn, kind='@edit')
487 if s is None:
488 return
489 encoding = 'utf-8' if e is None else e
490 # Delete all children.
491 while p.hasChildren():
492 p.firstChild().doDelete()
493 head = ''
494 ext = ext.lower()
495 if ext in ('.html', '.htm'):
496 head = '@language html\n'
497 elif ext in ('.txt', '.text'):
498 head = '@nocolor\n'
499 else:
500 language = ic.languageForExtension(ext)
501 if language and language != 'unknown_language':
502 head = f"@language {language}\n"
503 else:
504 head = '@nocolor\n'
505 p.b = head + g.toUnicode(s, encoding=encoding, reportErrors=True)
506 g.doHook('after-edit', p=p)
507 #@+node:ekr.20190201104956.1: *5* at.readOneAtAsisNode
508 def readOneAtAsisNode(self, fn, p): # pragma: no cover
509 """Read one @asis node. Used only by refresh-from-disk"""
510 at, c = self, self.c
511 # #1521 & #1341.
512 fn = g.fullPath(c, p)
513 junk, ext = g.os_path_splitext(fn)
514 # Remember the full fileName.
515 at.rememberReadPath(fn, p)
516 # if not g.unitTesting: g.es("reading: @asis %s" % (g.shortFileName(fn)))
517 s, e = g.readFileIntoString(fn, kind='@edit')
518 if s is None:
519 return
520 encoding = 'utf-8' if e is None else e
521 # Delete all children.
522 while p.hasChildren():
523 p.firstChild().doDelete()
524 old_body = p.b
525 p.b = g.toUnicode(s, encoding=encoding, reportErrors=True)
526 if not c.isChanged() and p.b != old_body:
527 c.setChanged()
528 #@+node:ekr.20150204165040.5: *5* at.readOneAtCleanNode & helpers
529 def readOneAtCleanNode(self, root): # pragma: no cover
530 """Update the @clean/@nosent node at root."""
531 at, c, x = self, self.c, self.c.shadowController
532 fileName = g.fullPath(c, root)
533 if not g.os_path_exists(fileName):
534 g.es_print(f"not found: {fileName}", color='red', nodeLink=root.get_UNL())
535 return False
536 at.rememberReadPath(fileName, root)
537 # Must be called before at.scanAllDirectives.
538 at.initReadIvars(root, fileName)
539 # Sets at.startSentinelComment/endSentinelComment.
540 at.scanAllDirectives(root)
541 new_public_lines = at.read_at_clean_lines(fileName)
542 old_private_lines = self.write_at_clean_sentinels(root)
543 marker = x.markerFromFileLines(old_private_lines, fileName)
544 old_public_lines, junk = x.separate_sentinels(old_private_lines, marker)
545 if old_public_lines:
546 new_private_lines = x.propagate_changed_lines(
547 new_public_lines, old_private_lines, marker, p=root)
548 else:
549 new_private_lines = []
550 root.b = ''.join(new_public_lines)
551 return True
552 if new_private_lines == old_private_lines:
553 return True
554 if not g.unitTesting:
555 g.es("updating:", root.h)
556 root.clearVisitedInTree()
557 gnx2vnode = at.fileCommands.gnxDict
558 contents = ''.join(new_private_lines)
559 FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
560 return True # Errors not detected.
561 #@+node:ekr.20150204165040.7: *6* at.dump_lines
562 def dump(self, lines, tag): # pragma: no cover
563 """Dump all lines."""
564 print(f"***** {tag} lines...\n")
565 for s in lines:
566 print(s.rstrip())
567 #@+node:ekr.20150204165040.8: *6* at.read_at_clean_lines
568 def read_at_clean_lines(self, fn): # pragma: no cover
569 """Return all lines of the @clean/@nosent file at fn."""
570 at = self
571 # Use the standard helper. Better error reporting.
572 # Important: uses 'rb' to open the file.
573 s = at.openFileHelper(fn)
574 # #1798.
575 if s is None:
576 s = ''
577 else:
578 s = g.toUnicode(s, encoding=at.encoding)
579 s = s.replace('\r\n', '\n') # Suppress meaningless "node changed" messages.
580 return g.splitLines(s)
581 #@+node:ekr.20150204165040.9: *6* at.write_at_clean_sentinels
582 def write_at_clean_sentinels(self, root): # pragma: no cover
583 """
584 Return all lines of the @clean tree as if it were
585 written as an @file node.
586 """
587 at = self
588 result = at.atFileToString(root, sentinels=True)
589 s = g.toUnicode(result, encoding=at.encoding)
590 return g.splitLines(s)
591 #@+node:ekr.20080711093251.7: *5* at.readOneAtShadowNode & helper
592 def readOneAtShadowNode(self, fn, p): # pragma: no cover
594 at, c = self, self.c
595 x = c.shadowController
596 if not fn == p.atShadowFileNodeName():
597 at.error(
598 f"can not happen: fn: {fn} != atShadowNodeName: "
599 f"{p.atShadowFileNodeName()}")
600 return
601 fn = g.fullPath(c, p) # #1521 & #1341.
602 # #889175: Remember the full fileName.
603 at.rememberReadPath(fn, p)
604 shadow_fn = x.shadowPathName(fn)
605 shadow_exists = g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn)
606 # Delete all children.
607 while p.hasChildren():
608 p.firstChild().doDelete()
609 if shadow_exists:
610 at.read(p)
611 else:
612 ok = at.importAtShadowNode(p)
613 if ok:
614 # Create the private file automatically.
615 at.writeOneAtShadowNode(p)
616 #@+node:ekr.20080712080505.1: *6* at.importAtShadowNode
617 def importAtShadowNode(self, p): # pragma: no cover
618 c, ic = self.c, self.c.importCommands
619 fn = g.fullPath(c, p) # #1521, #1341, #1914.
620 if not g.os_path_exists(fn):
621 g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
622 return p
623 # Delete all the child nodes.
624 while p.hasChildren():
625 p.firstChild().doDelete()
626 # Import the outline, exactly as @auto does.
627 ic.createOutline(parent=p.copy())
628 if ic.errors:
629 g.error('errors inhibited read @shadow', fn)
630 if ic.errors or not g.os_path_exists(fn):
631 p.clearDirty()
632 return ic.errors == 0
633 #@+node:ekr.20180622110112.1: *4* at.fast_read_into_root
634 def fast_read_into_root(self, c, contents, gnx2vnode, path, root): # pragma: no cover
635 """A convenience wrapper for FastAtRead.read_into_root()"""
636 return FastAtRead(c, gnx2vnode).read_into_root(contents, path, root)
637 #@+node:ekr.20041005105605.116: *4* at.Reading utils...
638 #@+node:ekr.20041005105605.119: *5* at.createImportedNode
639 def createImportedNode(self, root, headline): # pragma: no cover
640 at = self
641 if at.importRootSeen:
642 p = root.insertAsLastChild()
643 p.initHeadString(headline)
644 else:
645 # Put the text into the already-existing root node.
646 p = root
647 at.importRootSeen = True
648 p.v.setVisited() # Suppress warning about unvisited node.
649 return p
650 #@+node:ekr.20130911110233.11286: *5* at.initReadLine
651 def initReadLine(self, s):
652 """Init the ivars so that at.readLine will read all of s."""
653 at = self
654 at.read_i = 0
655 at.read_lines = g.splitLines(s)
656 at._file_bytes = g.toEncodedString(s)
657 #@+node:ekr.20041005105605.120: *5* at.parseLeoSentinel
658 def parseLeoSentinel(self, s):
659 """
660 Parse the sentinel line s.
661 If the sentinel is valid, set at.encoding, at.readVersion, at.readVersion5.
662 """
663 at, c = self, self.c
664 # Set defaults.
665 encoding = c.config.default_derived_file_encoding
666 readVersion, readVersion5 = None, None
667 new_df, start, end, isThin = False, '', '', False
668 # Example: \*@+leo-ver=5-thin-encoding=utf-8,.*/
669 pattern = re.compile(
670 r'(.+)@\+leo(-ver=([0123456789]+))?(-thin)?(-encoding=(.*)(\.))?(.*)')
671 # The old code weirdly allowed '.' in version numbers.
672 # group 1: opening delim
673 # group 2: -ver=
674 # group 3: version number
675 # group(4): -thin
676 # group(5): -encoding=utf-8,.
677 # group(6): utf-8,
678 # group(7): .
679 # group(8): closing delim.
680 m = pattern.match(s)
681 valid = bool(m)
682 if valid:
683 start = m.group(1) # start delim
684 valid = bool(start)
685 if valid:
686 new_df = bool(m.group(2)) # -ver=
687 if new_df:
688 # Set the version number.
689 if m.group(3):
690 readVersion = m.group(3)
691 readVersion5 = readVersion >= '5'
692 else:
693 valid = False # pragma: no cover
694 if valid:
695 # set isThin
696 isThin = bool(m.group(4))
697 if valid and m.group(5):
698 # set encoding.
699 encoding = m.group(6)
700 if encoding and encoding.endswith(','):
701 # Leo 4.2 or after.
702 encoding = encoding[:-1]
703 if not g.isValidEncoding(encoding): # pragma: no cover
704 g.es_print("bad encoding in derived file:", encoding)
705 valid = False
706 if valid:
707 end = m.group(8) # closing delim
708 if valid:
709 at.encoding = encoding
710 at.readVersion = readVersion
711 at.readVersion5 = readVersion5
712 return valid, new_df, start, end, isThin
713 #@+node:ekr.20130911110233.11284: *5* at.readFileToUnicode & helpers
714 def readFileToUnicode(self, fileName): # pragma: no cover
715 """
716 Carefully sets at.encoding, then uses at.encoding to convert the file
717 to a unicode string.
719 Sets at.encoding as follows:
720 1. Use the BOM, if present. This unambiguously determines the encoding.
721 2. Use the -encoding= field in the @+leo header, if present and valid.
722 3. Otherwise, uses existing value of at.encoding, which comes from:
723 A. An @encoding directive, found by at.scanAllDirectives.
724 B. The value of c.config.default_derived_file_encoding.
726 Returns the string, or None on failure.
727 """
728 at = self
729 s = at.openFileHelper(fileName) # Catches all exceptions.
730 # #1798.
731 if s is None:
732 return None
733 e, s = g.stripBOM(s)
734 if e:
735 # The BOM determines the encoding unambiguously.
736 s = g.toUnicode(s, encoding=e)
737 else:
738 # Get the encoding from the header, or the default encoding.
739 s_temp = g.toUnicode(s, 'ascii', reportErrors=False)
740 e = at.getEncodingFromHeader(fileName, s_temp)
741 s = g.toUnicode(s, encoding=e)
742 s = s.replace('\r\n', '\n')
743 at.encoding = e
744 at.initReadLine(s)
745 return s
746 #@+node:ekr.20130911110233.11285: *6* at.openFileHelper
747 def openFileHelper(self, fileName):
748 """Open a file, reporting all exceptions."""
749 at = self
750 # #1798: return None as a flag on any error.
751 s = None
752 try:
753 with open(fileName, 'rb') as f:
754 s = f.read()
755 except IOError: # pragma: no cover
756 at.error(f"can not open {fileName}")
757 except Exception: # pragma: no cover
758 at.error(f"Exception reading {fileName}")
759 g.es_exception()
760 return s
761 #@+node:ekr.20130911110233.11287: *6* at.getEncodingFromHeader
762 def getEncodingFromHeader(self, fileName, s):
763 """
764 Return the encoding given in the @+leo sentinel, if the sentinel is
765 present, or the previous value of at.encoding otherwise.
766 """
767 at = self
768 if at.errors: # pragma: no cover
769 g.trace('can not happen: at.errors > 0', g.callers())
770 e = at.encoding
771 if g.unitTesting:
772 assert False, g.callers()
773 else:
774 at.initReadLine(s)
775 old_encoding = at.encoding
776 assert old_encoding
777 at.encoding = None
778 # Execute scanHeader merely to set at.encoding.
779 at.scanHeader(fileName, giveErrors=False)
780 e = at.encoding or old_encoding
781 assert e
782 return e
783 #@+node:ekr.20041005105605.128: *5* at.readLine
784 def readLine(self):
785 """
786 Read one line from file using the present encoding.
787 Returns at.read_lines[at.read_i++]
788 """
789 # This is an old interface, now used only by at.scanHeader.
790 # For now, it's not worth replacing.
791 at = self
792 if at.read_i < len(at.read_lines):
793 s = at.read_lines[at.read_i]
794 at.read_i += 1
795 return s
796 # Not an error.
797 return '' # pragma: no cover
798 #@+node:ekr.20041005105605.129: *5* at.scanHeader
799 def scanHeader(self, fileName, giveErrors=True):
800 """
801 Scan the @+leo sentinel, using the old readLine interface.
803 Sets self.encoding, and self.start/endSentinelComment.
805 Returns (firstLines,new_df,isThinDerivedFile) where:
806 firstLines contains all @first lines,
807 new_df is True if we are reading a new-format derived file.
808 isThinDerivedFile is True if the file is an @thin file.
809 """
810 at = self
811 new_df, isThinDerivedFile = False, False
812 firstLines: List[str] = [] # The lines before @+leo.
813 s = self.scanFirstLines(firstLines)
814 valid = len(s) > 0
815 if valid:
816 valid, new_df, start, end, isThinDerivedFile = at.parseLeoSentinel(s)
817 if valid:
818 at.startSentinelComment = start
819 at.endSentinelComment = end
820 elif giveErrors: # pragma: no cover
821 at.error(f"No @+leo sentinel in: {fileName}")
822 g.trace(g.callers())
823 return firstLines, new_df, isThinDerivedFile
824 #@+node:ekr.20041005105605.130: *6* at.scanFirstLines
825 def scanFirstLines(self, firstLines): # pragma: no cover
826 """
827 Append all lines before the @+leo line to firstLines.
829 Empty lines are ignored because empty @first directives are
830 ignored.
832 We can not call sentinelKind here because that depends on the comment
833 delimiters we set here.
834 """
835 at = self
836 s = at.readLine()
837 while s and s.find("@+leo") == -1:
838 firstLines.append(s)
839 s = at.readLine()
840 return s
841 #@+node:ekr.20050103163224: *5* at.scanHeaderForThin (import code)
842 def scanHeaderForThin(self, fileName): # pragma: no cover
843 """
844 Return true if the derived file is a thin file.
846 This is a kludgy method used only by the import code."""
847 at = self
848 # Set at.encoding, regularize whitespace and call at.initReadLines.
849 at.readFileToUnicode(fileName)
850 # scanHeader uses at.readline instead of its args.
851 # scanHeader also sets at.encoding.
852 junk, junk, isThin = at.scanHeader(None)
853 return isThin
854 #@+node:ekr.20041005105605.132: *3* at.Writing
855 #@+node:ekr.20041005105605.133: *4* Writing (top level)
856 #@+node:ekr.20190111153551.1: *5* at.commands
857 #@+node:ekr.20070806105859: *6* at.writeAtAutoNodes
858 @cmd('write-at-auto-nodes')
859 def writeAtAutoNodes(self, event=None): # pragma: no cover
860 """Write all @auto nodes in the selected outline."""
861 at, c, p = self, self.c, self.c.p
862 c.init_error_dialogs()
863 after, found = p.nodeAfterTree(), False
864 while p and p != after:
865 if p.isAtAutoNode() and not p.isAtIgnoreNode():
866 ok = at.writeOneAtAutoNode(p)
867 if ok:
868 found = True
869 p.moveToNodeAfterTree()
870 else:
871 p.moveToThreadNext()
872 else:
873 p.moveToThreadNext()
874 if g.unitTesting:
875 return
876 if found:
877 g.es("finished")
878 else:
879 g.es("no @auto nodes in the selected tree")
880 c.raise_error_dialogs(kind='write')
882 #@+node:ekr.20220120072251.1: *6* at.writeDirtyAtAutoNodes
883 @cmd('write-dirty-at-auto-nodes') # pragma: no cover
884 def writeDirtyAtAutoNodes(self, event=None):
885 """Write all dirty @auto nodes in the selected outline."""
886 at, c, p = self, self.c, self.c.p
887 c.init_error_dialogs()
888 after, found = p.nodeAfterTree(), False
889 while p and p != after:
890 if p.isAtAutoNode() and not p.isAtIgnoreNode() and p.isDirty():
891 ok = at.writeOneAtAutoNode(p)
892 if ok:
893 found = True
894 p.moveToNodeAfterTree()
895 else:
896 p.moveToThreadNext()
897 else:
898 p.moveToThreadNext()
899 if g.unitTesting:
900 return
901 if found:
902 g.es("finished")
903 else:
904 g.es("no dirty @auto nodes in the selected tree")
905 c.raise_error_dialogs(kind='write')
906 #@+node:ekr.20080711093251.3: *6* at.writeAtShadowNodes
907 @cmd('write-at-shadow-nodes')
908 def writeAtShadowNodes(self, event=None): # pragma: no cover
909 """Write all @shadow nodes in the selected outline."""
910 at, c, p = self, self.c, self.c.p
911 c.init_error_dialogs()
912 after, found = p.nodeAfterTree(), False
913 while p and p != after:
914 if p.atShadowFileNodeName() and not p.isAtIgnoreNode():
915 ok = at.writeOneAtShadowNode(p)
916 if ok:
917 found = True
918 g.blue(f"wrote {p.atShadowFileNodeName()}")
919 p.moveToNodeAfterTree()
920 else:
921 p.moveToThreadNext()
922 else:
923 p.moveToThreadNext()
924 if g.unitTesting:
925 return found
926 if found:
927 g.es("finished")
928 else:
929 g.es("no @shadow nodes in the selected tree")
930 c.raise_error_dialogs(kind='write')
931 return found
933 #@+node:ekr.20220120072917.1: *6* at.writeDirtyAtShadowNodes
934 @cmd('write-dirty-at-shadow-nodes')
935 def writeDirtyAtShadowNodes(self, event=None): # pragma: no cover
936 """Write all @shadow nodes in the selected outline."""
937 at, c, p = self, self.c, self.c.p
938 c.init_error_dialogs()
939 after, found = p.nodeAfterTree(), False
940 while p and p != after:
941 if p.atShadowFileNodeName() and not p.isAtIgnoreNode() and p.isDirty():
942 ok = at.writeOneAtShadowNode(p)
943 if ok:
944 found = True
945 g.blue(f"wrote {p.atShadowFileNodeName()}")
946 p.moveToNodeAfterTree()
947 else:
948 p.moveToThreadNext()
949 else:
950 p.moveToThreadNext()
951 if g.unitTesting:
952 return found
953 if found:
954 g.es("finished")
955 else:
956 g.es("no dirty @shadow nodes in the selected tree")
957 c.raise_error_dialogs(kind='write')
958 return found
960 #@+node:ekr.20041005105605.157: *5* at.putFile
961 def putFile(self, root, fromString='', sentinels=True):
962 """Write the contents of the file to the output stream."""
963 at = self
964 s = fromString if fromString else root.v.b
965 root.clearAllVisitedInTree()
966 at.putAtFirstLines(s)
967 at.putOpenLeoSentinel("@+leo-ver=5")
968 at.putInitialComment()
969 at.putOpenNodeSentinel(root)
970 at.putBody(root, fromString=fromString)
971 # The -leo sentinel is required to handle @last.
972 at.putSentinel("@-leo")
973 root.setVisited()
974 at.putAtLastLines(s)
975 #@+node:ekr.20041005105605.147: *5* at.writeAll & helpers
976 def writeAll(self, all=False, dirty=False):
977 """Write @file nodes in all or part of the outline"""
978 at = self
979 # This is the *only* place where these are set.
980 # promptForDangerousWrite sets cancelFlag only if canCancelFlag is True.
981 at.unchangedFiles = 0
982 at.canCancelFlag = True
983 at.cancelFlag = False
984 at.yesToAll = False
985 files, root = at.findFilesToWrite(all)
986 for p in files:
987 try:
988 at.writeAllHelper(p, root)
989 except Exception: # pragma: no cover
990 at.internalWriteError(p)
991 # Make *sure* these flags are cleared for other commands.
992 at.canCancelFlag = False
993 at.cancelFlag = False
994 at.yesToAll = False
995 # Say the command is finished.
996 at.reportEndOfWrite(files, all, dirty)
997 # #2338: Never call at.saveOutlineIfPossible().
998 #@+node:ekr.20190108052043.1: *6* at.findFilesToWrite
999 def findFilesToWrite(self, force): # pragma: no cover
1000 """
1001 Return a list of files to write.
1002 We must do this in a prepass, so as to avoid errors later.
1003 """
1004 trace = 'save' in g.app.debug and not g.unitTesting
1005 if trace:
1006 g.trace(f"writing *{'selected' if force else 'all'}* files")
1007 c = self.c
1008 if force:
1009 # The Write @<file> Nodes command.
1010 # Write all nodes in the selected tree.
1011 root = c.p
1012 p = c.p
1013 after = p.nodeAfterTree()
1014 else:
1015 # Write dirty nodes in the entire outline.
1016 root = c.rootPosition()
1017 p = c.rootPosition()
1018 after = None
1019 seen = set()
1020 files = []
1021 while p and p != after:
1022 if p.isAtIgnoreNode() and not p.isAtAsisFileNode():
1023 # Honor @ignore in *body* text, but *not* in @asis nodes.
1024 if p.isAnyAtFileNode():
1025 c.ignored_at_file_nodes.append(p.h)
1026 p.moveToNodeAfterTree()
1027 elif p.isAnyAtFileNode():
1028 data = p.v, g.fullPath(c, p)
1029 if data in seen:
1030 if trace and force:
1031 g.trace('Already seen', p.h)
1032 else:
1033 seen.add(data)
1034 files.append(p.copy())
1035 # Don't scan nested trees???
1036 p.moveToNodeAfterTree()
1037 else:
1038 p.moveToThreadNext()
1039 # When scanning *all* nodes, we only actually write dirty nodes.
1040 if not force:
1041 files = [z for z in files if z.isDirty()]
1042 if trace:
1043 g.printObj([z.h for z in files], tag='Files to be saved')
1044 return files, root
1045 #@+node:ekr.20190108053115.1: *6* at.internalWriteError
1046 def internalWriteError(self, p): # pragma: no cover
1047 """
1048 Fix bug 1260415: https://bugs.launchpad.net/leo-editor/+bug/1260415
1049 Give a more urgent, more specific, more helpful message.
1050 """
1051 g.es_exception()
1052 g.es(f"Internal error writing: {p.h}", color='red')
1053 g.es('Please report this error to:', color='blue')
1054 g.es('https://groups.google.com/forum/#!forum/leo-editor', color='blue')
1055 g.es('Warning: changes to this file will be lost', color='red')
1056 g.es('unless you can save the file successfully.', color='red')
1057 #@+node:ekr.20190108112519.1: *6* at.reportEndOfWrite
1058 def reportEndOfWrite(self, files, all, dirty): # pragma: no cover
1060 at = self
1061 if g.unitTesting:
1062 return
1063 if files:
1064 n = at.unchangedFiles
1065 g.es(f"finished: {n} unchanged file{g.plural(n)}")
1066 elif all:
1067 g.warning("no @<file> nodes in the selected tree")
1068 elif dirty:
1069 g.es("no dirty @<file> nodes in the selected tree")
1070 #@+node:ekr.20041005105605.149: *6* at.writeAllHelper & helper
1071 def writeAllHelper(self, p, root):
1072 """
1073 Write one file for at.writeAll.
1075 Do *not* write @auto files unless p == root.
1077 This prevents the write-all command from needlessly updating
1078 the @persistence data, thereby annoyingly changing the .leo file.
1079 """
1080 at = self
1081 at.root = root
1082 if p.isAtIgnoreNode(): # pragma: no cover
1083 # Should have been handled in findFilesToWrite.
1084 g.trace(f"Can not happen: {p.h} is an @ignore node")
1085 return
1086 try:
1087 at.writePathChanged(p)
1088 except IOError: # pragma: no cover
1089 return
1090 table = (
1091 (p.isAtAsisFileNode, at.asisWrite),
1092 (p.isAtAutoNode, at.writeOneAtAutoNode),
1093 (p.isAtCleanNode, at.writeOneAtCleanNode),
1094 (p.isAtEditNode, at.writeOneAtEditNode),
1095 (p.isAtFileNode, at.writeOneAtFileNode),
1096 (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
1097 (p.isAtShadowFileNode, at.writeOneAtShadowNode),
1098 (p.isAtThinFileNode, at.writeOneAtFileNode),
1099 )
1100 for pred, func in table:
1101 if pred():
1102 func(p) # type:ignore
1103 break
1104 else: # pragma: no cover
1105 g.trace(f"Can not happen: {p.h}")
1106 return
1107 #
1108 # Clear the dirty bits in all descendant nodes.
1109 # The persistence data may still have to be written.
1110 for p2 in p.self_and_subtree(copy=False):
1111 p2.v.clearDirty()
1112 #@+node:ekr.20190108105509.1: *7* at.writePathChanged
1113 def writePathChanged(self, p): # pragma: no cover
1114 """
1115 raise IOError if p's path has changed *and* user forbids the write.
1116 """
1117 at, c = self, self.c
1118 #
1119 # Suppress this message during save-as and save-to commands.
1120 if c.ignoreChangedPaths:
1121 return # pragma: no cover
1122 oldPath = g.os_path_normcase(at.getPathUa(p))
1123 newPath = g.os_path_normcase(g.fullPath(c, p))
1124 try: # #1367: samefile can throw an exception.
1125 changed = oldPath and not os.path.samefile(oldPath, newPath)
1126 except Exception:
1127 changed = True
1128 if not changed:
1129 return
1130 ok = at.promptForDangerousWrite(
1131 fileName=None,
1132 message=(
1133 f"{g.tr('path changed for %s' % (p.h))}\n"
1134 f"{g.tr('write this file anyway?')}"
1135 ),
1136 )
1137 if not ok:
1138 raise IOError
1139 at.setPathUa(p, newPath) # Remember that we have changed paths.
1140 #@+node:ekr.20190109172025.1: *5* at.writeAtAutoContents
1141 def writeAtAutoContents(self, fileName, root): # pragma: no cover
1142 """Common helper for atAutoToString and writeOneAtAutoNode."""
1143 at, c = self, self.c
1144 # Dispatch the proper writer.
1145 junk, ext = g.os_path_splitext(fileName)
1146 writer = at.dispatch(ext, root)
1147 if writer:
1148 at.outputList = []
1149 writer(root)
1150 return '' if at.errors else ''.join(at.outputList)
1151 if root.isAtAutoRstNode():
1152 # An escape hatch: fall back to the theRst writer
1153 # if there is no rst writer plugin.
1154 at.outputFile = outputFile = io.StringIO()
1155 ok = c.rstCommands.writeAtAutoFile(root, fileName, outputFile)
1156 return outputFile.close() if ok else None
1157 # leo 5.6: allow undefined section references in all @auto files.
1158 ivar = 'allow_undefined_refs'
1159 try:
1160 setattr(at, ivar, True)
1161 at.outputList = []
1162 at.putFile(root, sentinels=False)
1163 return '' if at.errors else ''.join(at.outputList)
1164 except Exception:
1165 return None
1166 finally:
1167 if hasattr(at, ivar):
1168 delattr(at, ivar)
1169 #@+node:ekr.20190111153522.1: *5* at.writeX...
1170 #@+node:ekr.20041005105605.154: *6* at.asisWrite & helper
1171 def asisWrite(self, root): # pragma: no cover
1172 at, c = self, self.c
1173 try:
1174 c.endEditing()
1175 c.init_error_dialogs()
1176 fileName = at.initWriteIvars(root)
1177 # #1450.
1178 if not fileName or not at.precheck(fileName, root):
1179 at.addToOrphanList(root)
1180 return
1181 at.outputList = []
1182 for p in root.self_and_subtree(copy=False):
1183 at.writeAsisNode(p)
1184 if not at.errors:
1185 contents = ''.join(at.outputList)
1186 at.replaceFile(contents, at.encoding, fileName, root)
1187 except Exception:
1188 at.writeException(fileName, root)
1190 silentWrite = asisWrite # Compatibility with old scripts.
1191 #@+node:ekr.20170331141933.1: *7* at.writeAsisNode
1192 def writeAsisNode(self, p): # pragma: no cover
1193 """Write the p's node to an @asis file."""
1194 at = self
1196 def put(s):
1197 """Append s to self.output_list."""
1198 # #1480: Avoid calling at.os().
1199 s = g.toUnicode(s, at.encoding, reportErrors=True)
1200 at.outputList.append(s)
1202 # Write the headline only if it starts with '@@'.
1204 s = p.h
1205 if g.match(s, 0, "@@"):
1206 s = s[2:]
1207 if s:
1208 put('\n') # Experimental.
1209 put(s)
1210 put('\n')
1211 # Write the body.
1212 s = p.b
1213 if s:
1214 put(s)
1215 #@+node:ekr.20041005105605.151: *6* at.writeMissing & helper
1216 def writeMissing(self, p): # pragma: no cover
1217 at, c = self, self.c
1218 writtenFiles = False
1219 c.init_error_dialogs()
1220 # #1450.
1221 at.initWriteIvars(root=p.copy())
1222 p = p.copy()
1223 after = p.nodeAfterTree()
1224 while p and p != after: # Don't use iterator.
1225 if (
1226 p.isAtAsisFileNode() or (p.isAnyAtFileNode() and not p.isAtIgnoreNode())
1227 ):
1228 fileName = p.anyAtFileNodeName()
1229 if fileName:
1230 fileName = g.fullPath(c, p) # #1914.
1231 if at.precheck(fileName, p):
1232 at.writeMissingNode(p)
1233 writtenFiles = True
1234 else:
1235 at.addToOrphanList(p)
1236 p.moveToNodeAfterTree()
1237 elif p.isAtIgnoreNode():
1238 p.moveToNodeAfterTree()
1239 else:
1240 p.moveToThreadNext()
1241 if not g.unitTesting:
1242 if writtenFiles > 0:
1243 g.es("finished")
1244 else:
1245 g.es("no @file node in the selected tree")
1246 c.raise_error_dialogs(kind='write')
1247 #@+node:ekr.20041005105605.152: *7* at.writeMissingNode
1248 def writeMissingNode(self, p): # pragma: no cover
1250 at = self
1251 table = (
1252 (p.isAtAsisFileNode, at.asisWrite),
1253 (p.isAtAutoNode, at.writeOneAtAutoNode),
1254 (p.isAtCleanNode, at.writeOneAtCleanNode),
1255 (p.isAtEditNode, at.writeOneAtEditNode),
1256 (p.isAtFileNode, at.writeOneAtFileNode),
1257 (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
1258 (p.isAtShadowFileNode, at.writeOneAtShadowNode),
1259 (p.isAtThinFileNode, at.writeOneAtFileNode),
1260 )
1261 for pred, func in table:
1262 if pred():
1263 func(p) # type:ignore
1264 return
1265 g.trace(f"Can not happen unknown @<file> kind: {p.h}")
1266 #@+node:ekr.20070806141607: *6* at.writeOneAtAutoNode & helpers
1267 def writeOneAtAutoNode(self, p): # pragma: no cover
1268 """
1269 Write p, an @auto node.
1270 File indices *must* have already been assigned.
1271 Return True if the node was written successfully.
1272 """
1273 at, c = self, self.c
1274 root = p.copy()
1275 try:
1276 c.endEditing()
1277 if not p.atAutoNodeName():
1278 return False
1279 fileName = at.initWriteIvars(root)
1280 at.sentinels = False
1281 # #1450.
1282 if not fileName or not at.precheck(fileName, root):
1283 at.addToOrphanList(root)
1284 return False
1285 if c.persistenceController:
1286 c.persistenceController.update_before_write_foreign_file(root)
1287 contents = at.writeAtAutoContents(fileName, root)
1288 if contents is None:
1289 g.es("not written:", fileName)
1290 at.addToOrphanList(root)
1291 return False
1292 at.replaceFile(contents, at.encoding, fileName, root,
1293 ignoreBlankLines=root.isAtAutoRstNode())
1294 return True
1295 except Exception:
1296 at.writeException(fileName, root)
1297 return False
1298 #@+node:ekr.20140728040812.17993: *7* at.dispatch & helpers
1299 def dispatch(self, ext, p): # pragma: no cover
1300 """Return the correct writer function for p, an @auto node."""
1301 at = self
1302 # Match @auto type before matching extension.
1303 return at.writer_for_at_auto(p) or at.writer_for_ext(ext)
1304 #@+node:ekr.20140728040812.17995: *8* at.writer_for_at_auto
1305 def writer_for_at_auto(self, root): # pragma: no cover
1306 """A factory returning a writer function for the given kind of @auto directive."""
1307 at = self
1308 d = g.app.atAutoWritersDict
1309 for key in d:
1310 aClass = d.get(key)
1311 if aClass and g.match_word(root.h, 0, key):
1313 def writer_for_at_auto_cb(root):
1314 # pylint: disable=cell-var-from-loop
1315 try:
1316 writer = aClass(at.c)
1317 s = writer.write(root)
1318 return s
1319 except Exception:
1320 g.es_exception()
1321 return None
1323 return writer_for_at_auto_cb
1324 return None
1325 #@+node:ekr.20140728040812.17997: *8* at.writer_for_ext
1326 def writer_for_ext(self, ext): # pragma: no cover
1327 """A factory returning a writer function for the given file extension."""
1328 at = self
1329 d = g.app.writersDispatchDict
1330 aClass = d.get(ext)
1331 if aClass:
1333 def writer_for_ext_cb(root):
1334 try:
1335 return aClass(at.c).write(root)
1336 except Exception:
1337 g.es_exception()
1338 return None
1340 return writer_for_ext_cb
1342 return None
1343 #@+node:ekr.20210501064359.1: *6* at.writeOneAtCleanNode
1344 def writeOneAtCleanNode(self, root): # pragma: no cover
1345 """Write one @clean file..
1346 root is the position of an @clean node.
1347 """
1348 at, c = self, self.c
1349 try:
1350 c.endEditing()
1351 fileName = at.initWriteIvars(root)
1352 at.sentinels = False
1353 if not fileName or not at.precheck(fileName, root):
1354 return
1355 at.outputList = []
1356 at.putFile(root, sentinels=False)
1357 at.warnAboutOrphandAndIgnoredNodes()
1358 if at.errors:
1359 g.es("not written:", g.shortFileName(fileName))
1360 at.addToOrphanList(root)
1361 else:
1362 contents = ''.join(at.outputList)
1363 at.replaceFile(contents, at.encoding, fileName, root)
1364 except Exception:
1365 at.writeException(fileName, root)
1366 #@+node:ekr.20090225080846.5: *6* at.writeOneAtEditNode
1367 def writeOneAtEditNode(self, p): # pragma: no cover
1368 """Write one @edit node."""
1369 at, c = self, self.c
1370 root = p.copy()
1371 try:
1372 c.endEditing()
1373 c.init_error_dialogs()
1374 if not p.atEditNodeName():
1375 return False
1376 if p.hasChildren():
1377 g.error('@edit nodes must not have children')
1378 g.es('To save your work, convert @edit to @auto, @file or @clean')
1379 return False
1380 fileName = at.initWriteIvars(root)
1381 at.sentinels = False
1382 # #1450.
1383 if not fileName or not at.precheck(fileName, root):
1384 at.addToOrphanList(root)
1385 return False
1386 contents = ''.join([s for s in g.splitLines(p.b)
1387 if at.directiveKind4(s, 0) == at.noDirective])
1388 at.replaceFile(contents, at.encoding, fileName, root)
1389 c.raise_error_dialogs(kind='write')
1390 return True
1391 except Exception:
1392 at.writeException(fileName, root)
1393 return False
1394 #@+node:ekr.20210501075610.1: *6* at.writeOneAtFileNode
1395 def writeOneAtFileNode(self, root): # pragma: no cover
1396 """Write @file or @thin file."""
1397 at, c = self, self.c
1398 try:
1399 c.endEditing()
1400 fileName = at.initWriteIvars(root)
1401 at.sentinels = True
1402 if not fileName or not at.precheck(fileName, root):
1403 # Raise dialog warning of data loss.
1404 at.addToOrphanList(root)
1405 return
1406 at.outputList = []
1407 at.putFile(root, sentinels=True)
1408 at.warnAboutOrphandAndIgnoredNodes()
1409 if at.errors:
1410 g.es("not written:", g.shortFileName(fileName))
1411 at.addToOrphanList(root)
1412 else:
1413 contents = ''.join(at.outputList)
1414 at.replaceFile(contents, at.encoding, fileName, root)
1415 except Exception:
1416 at.writeException(fileName, root)
1417 #@+node:ekr.20210501065352.1: *6* at.writeOneAtNosentNode
1418 def writeOneAtNosentNode(self, root): # pragma: no cover
1419 """Write one @nosent node.
1420 root is the position of an @<file> node.
1421 sentinels will be False for @clean and @nosent nodes.
1422 """
1423 at, c = self, self.c
1424 try:
1425 c.endEditing()
1426 fileName = at.initWriteIvars(root)
1427 at.sentinels = False
1428 if not fileName or not at.precheck(fileName, root):
1429 return
1430 at.outputList = []
1431 at.putFile(root, sentinels=False)
1432 at.warnAboutOrphandAndIgnoredNodes()
1433 if at.errors:
1434 g.es("not written:", g.shortFileName(fileName))
1435 at.addToOrphanList(root)
1436 else:
1437 contents = ''.join(at.outputList)
1438 at.replaceFile(contents, at.encoding, fileName, root)
1439 except Exception:
1440 at.writeException(fileName, root)
1441 #@+node:ekr.20080711093251.5: *6* at.writeOneAtShadowNode & helper
1442 def writeOneAtShadowNode(self, p, testing=False): # pragma: no cover
1443 """
1444 Write p, an @shadow node.
1445 File indices *must* have already been assigned.
1447 testing: set by unit tests to suppress the call to at.precheck.
1448 Testing is not the same as g.unitTesting.
1449 """
1450 at, c = self, self.c
1451 root = p.copy()
1452 x = c.shadowController
1453 try:
1454 c.endEditing() # Capture the current headline.
1455 fn = p.atShadowFileNodeName()
1456 assert fn, p.h
1457 # A hack to support unknown extensions. May set c.target_language.
1458 self.adjustTargetLanguage(fn)
1459 full_path = g.fullPath(c, p)
1460 at.initWriteIvars(root)
1461 # Force python sentinels to suppress an error message.
1462 # The actual sentinels will be set below.
1463 at.endSentinelComment = None
1464 at.startSentinelComment = "#"
1465 # Make sure we can compute the shadow directory.
1466 private_fn = x.shadowPathName(full_path)
1467 if not private_fn:
1468 return False
1469 if not testing and not at.precheck(full_path, root):
1470 return False
1471 #
1472 # Bug fix: Leo 4.5.1:
1473 # use x.markerFromFileName to force the delim to match
1474 # what is used in x.propegate changes.
1475 marker = x.markerFromFileName(full_path)
1476 at.startSentinelComment, at.endSentinelComment = marker.getDelims()
1477 if g.unitTesting:
1478 ivars_dict = g.getIvarsDict(at)
1479 #
1480 # Write the public and private files to strings.
1482 def put(sentinels):
1483 at.outputList = []
1484 at.sentinels = sentinels
1485 at.putFile(root, sentinels=sentinels)
1486 return '' if at.errors else ''.join(at.outputList)
1488 at.public_s = put(False)
1489 at.private_s = put(True)
1490 at.warnAboutOrphandAndIgnoredNodes()
1491 if g.unitTesting:
1492 exceptions = ('public_s', 'private_s', 'sentinels', 'outputList')
1493 assert g.checkUnchangedIvars(
1494 at, ivars_dict, exceptions), 'writeOneAtShadowNode'
1495 if not at.errors:
1496 # Write the public and private files.
1497 # makeShadowDirectory takes a *public* file name.
1498 x.makeShadowDirectory(full_path)
1499 x.replaceFileWithString(at.encoding, private_fn, at.private_s)
1500 x.replaceFileWithString(at.encoding, full_path, at.public_s)
1501 at.checkPythonCode(contents=at.private_s, fileName=full_path, root=root)
1502 if at.errors:
1503 g.error("not written:", full_path)
1504 at.addToOrphanList(root)
1505 else:
1506 root.clearDirty()
1507 return not at.errors
1508 except Exception:
1509 at.writeException(full_path, root)
1510 return False
1511 #@+node:ekr.20080819075811.13: *7* at.adjustTargetLanguage
1512 def adjustTargetLanguage(self, fn): # pragma: no cover
1513 """Use the language implied by fn's extension if
1514 there is a conflict between it and c.target_language."""
1515 at = self
1516 c = at.c
1517 junk, ext = g.os_path_splitext(fn)
1518 if ext:
1519 if ext.startswith('.'):
1520 ext = ext[1:]
1521 language = g.app.extension_dict.get(ext)
1522 if language:
1523 c.target_language = language
1524 else:
1525 # An unknown language.
1526 # Use the default language, **not** 'unknown_language'
1527 pass
1528 #@+node:ekr.20190111153506.1: *5* at.XToString
1529 #@+node:ekr.20190109160056.1: *6* at.atAsisToString
1530 def atAsisToString(self, root): # pragma: no cover
1531 """Write the @asis node to a string."""
1532 # pylint: disable=used-before-assignment
1533 at, c = self, self.c
1534 try:
1535 c.endEditing()
1536 fileName = at.initWriteIvars(root)
1537 at.outputList = []
1538 for p in root.self_and_subtree(copy=False):
1539 at.writeAsisNode(p)
1540 return '' if at.errors else ''.join(at.outputList)
1541 except Exception:
1542 at.writeException(fileName, root)
1543 return ''
1544 #@+node:ekr.20190109160056.2: *6* at.atAutoToString
1545 def atAutoToString(self, root): # pragma: no cover
1546 """Write the root @auto node to a string, and return it."""
1547 at, c = self, self.c
1548 try:
1549 c.endEditing()
1550 fileName = at.initWriteIvars(root)
1551 at.sentinels = False
1552 # #1450.
1553 if not fileName:
1554 at.addToOrphanList(root)
1555 return ''
1556 return at.writeAtAutoContents(fileName, root) or ''
1557 except Exception:
1558 at.writeException(fileName, root)
1559 return ''
1560 #@+node:ekr.20190109160056.3: *6* at.atEditToString
1561 def atEditToString(self, root): # pragma: no cover
1562 """Write one @edit node."""
1563 at, c = self, self.c
1564 try:
1565 c.endEditing()
1566 if root.hasChildren():
1567 g.error('@edit nodes must not have children')
1568 g.es('To save your work, convert @edit to @auto, @file or @clean')
1569 return False
1570 fileName = at.initWriteIvars(root)
1571 at.sentinels = False
1572 # #1450.
1573 if not fileName:
1574 at.addToOrphanList(root)
1575 return ''
1576 contents = ''.join([
1577 s for s in g.splitLines(root.b)
1578 if at.directiveKind4(s, 0) == at.noDirective])
1579 return contents
1580 except Exception:
1581 at.writeException(fileName, root)
1582 return ''
1583 #@+node:ekr.20190109142026.1: *6* at.atFileToString
1584 def atFileToString(self, root, sentinels=True): # pragma: no cover
1585 """Write an external file to a string, and return its contents."""
1586 at, c = self, self.c
1587 try:
1588 c.endEditing()
1589 at.initWriteIvars(root)
1590 at.sentinels = sentinels
1591 at.outputList = []
1592 at.putFile(root, sentinels=sentinels)
1593 assert root == at.root, 'write'
1594 contents = '' if at.errors else ''.join(at.outputList)
1595 return contents
1596 except Exception:
1597 at.exception("exception preprocessing script")
1598 root.v._p_changed = True
1599 return ''
1600 #@+node:ekr.20050506084734: *6* at.stringToString
1601 def stringToString(self, root, s, forcePythonSentinels=True, sentinels=True): # pragma: no cover
1602 """
1603 Write an external file from a string.
1605 This is at.write specialized for scripting.
1606 """
1607 at, c = self, self.c
1608 try:
1609 c.endEditing()
1610 at.initWriteIvars(root)
1611 if forcePythonSentinels:
1612 at.endSentinelComment = None
1613 at.startSentinelComment = "#"
1614 at.language = "python"
1615 at.sentinels = sentinels
1616 at.outputList = []
1617 at.putFile(root, fromString=s, sentinels=sentinels)
1618 contents = '' if at.errors else ''.join(at.outputList)
1619 # Major bug: failure to clear this wipes out headlines!
1620 # Sometimes this causes slight problems...
1621 if root:
1622 root.v._p_changed = True
1623 return contents
1624 except Exception:
1625 at.exception("exception preprocessing script")
1626 return ''
1627 #@+node:ekr.20041005105605.160: *4* Writing helpers
1628 #@+node:ekr.20041005105605.161: *5* at.putBody & helper
1629 def putBody(self, p, fromString=''):
1630 """
1631 Generate the body enclosed in sentinel lines.
1632 Return True if the body contains an @others line.
1633 """
1634 at = self
1635 #
1636 # New in 4.3 b2: get s from fromString if possible.
1637 s = fromString if fromString else p.b
1638 # Make sure v is never expanded again.
1639 # Suppress orphans check.
1640 p.v.setVisited()
1641 #
1642 # #1048 & #1037: regularize most trailing whitespace.
1643 if s and (at.sentinels or at.force_newlines_in_at_nosent_bodies):
1644 if not s.endswith('\n'):
1645 s = s + '\n'
1648 class Status:
1649 at_comment_seen = False
1650 at_delims_seen = False
1651 at_warning_given = False
1652 has_at_others = False
1653 in_code = True
1656 i = 0
1657 status = Status()
1658 while i < len(s):
1659 next_i = g.skip_line(s, i)
1660 assert next_i > i, 'putBody'
1661 kind = at.directiveKind4(s, i)
1662 at.putLine(i, kind, p, s, status)
1663 i = next_i
1664 if not status.in_code:
1665 at.putEndDocLine()
1666 return status.has_at_others
1667 #@+node:ekr.20041005105605.163: *6* at.putLine
1668 def putLine(self, i, kind, p, s, status):
1669 """Put the line at s[i:] of the given kind, updating the status."""
1670 at = self
1671 if kind == at.noDirective:
1672 if status.in_code:
1673 # Important: the so-called "name" must include brackets.
1674 name, n1, n2 = at.findSectionName(s, i, p)
1675 if name:
1676 at.putRefLine(s, i, n1, n2, name, p)
1677 else:
1678 at.putCodeLine(s, i)
1679 else:
1680 at.putDocLine(s, i)
1681 elif kind in (at.docDirective, at.atDirective):
1682 if not status.in_code:
1683 # Bug fix 12/31/04: handle adjacent doc parts.
1684 at.putEndDocLine()
1685 at.putStartDocLine(s, i, kind)
1686 status.in_code = False
1687 elif kind in (at.cDirective, at.codeDirective):
1688 # Only @c and @code end a doc part.
1689 if not status.in_code:
1690 at.putEndDocLine()
1691 at.putDirective(s, i, p)
1692 status.in_code = True
1693 elif kind == at.allDirective:
1694 if status.in_code:
1695 if p == self.root:
1696 at.putAtAllLine(s, i, p)
1697 else:
1698 at.error(f"@all not valid in: {p.h}") # pragma: no cover
1699 else:
1700 at.putDocLine(s, i)
1701 elif kind == at.othersDirective:
1702 if status.in_code:
1703 if status.has_at_others:
1704 at.error(f"multiple @others in: {p.h}") # pragma: no cover
1705 else:
1706 at.putAtOthersLine(s, i, p)
1707 status.has_at_others = True
1708 else:
1709 at.putDocLine(s, i)
1710 elif kind == at.startVerbatim: # pragma: no cover
1711 # Fix bug 778204: @verbatim not a valid Leo directive.
1712 if g.unitTesting:
1713 # A hack: unit tests for @shadow use @verbatim as a kind of directive.
1714 pass
1715 else:
1716 at.error(f"@verbatim is not a Leo directive: {p.h}")
1717 elif kind == at.miscDirective:
1718 # Fix bug 583878: Leo should warn about @comment/@delims clashes.
1719 if g.match_word(s, i, '@comment'):
1720 status.at_comment_seen = True
1721 elif g.match_word(s, i, '@delims'):
1722 status.at_delims_seen = True
1723 if (
1724 status.at_comment_seen and
1725 status.at_delims_seen and not
1726 status.at_warning_given
1727 ): # pragma: no cover
1728 status.at_warning_given = True
1729 at.error(f"@comment and @delims in node {p.h}")
1730 at.putDirective(s, i, p)
1731 else:
1732 at.error(f"putBody: can not happen: unknown directive kind: {kind}") # pragma: no cover
1733 #@+node:ekr.20041005105605.164: *5* writing code lines...
1734 #@+node:ekr.20041005105605.165: *6* at: @all
1735 #@+node:ekr.20041005105605.166: *7* at.putAtAllLine
1736 def putAtAllLine(self, s, i, p):
1737 """Put the expansion of @all."""
1738 at = self
1739 j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1740 k = g.skip_to_end_of_line(s, i)
1741 at.putLeadInSentinel(s, i, j)
1742 at.indent += delta
1743 # s[j:k] starts with '@all'
1744 at.putSentinel("@+" + s[j + 1 : k].strip())
1745 for child in p.children():
1746 at.putAtAllChild(child)
1747 at.putSentinel("@-all")
1748 at.indent -= delta
1749 #@+node:ekr.20041005105605.167: *7* at.putAtAllBody
1750 def putAtAllBody(self, p):
1751 """ Generate the body enclosed in sentinel lines."""
1752 at = self
1753 s = p.b
1754 # Make sure v is never expanded again.
1755 # Suppress orphans check.
1756 p.v.setVisited()
1757 if at.sentinels and s and s[-1] != '\n':
1758 s = s + '\n'
1759 i = 0
1760 # Leo 6.6. This code never changes at.in_code status!
1761 while i < len(s):
1762 next_i = g.skip_line(s, i)
1763 assert next_i > i
1764 at.putCodeLine(s, i)
1765 i = next_i
1766 #@+node:ekr.20041005105605.169: *7* at.putAtAllChild
1767 def putAtAllChild(self, p):
1768 """
1769 This code puts only the first of two or more cloned siblings, preceding
1770 the clone with an @clone n sentinel.
1772 This is a debatable choice: the cloned tree appears only once in the
1773 external file. This should be benign; the text created by @all is
1774 likely to be used only for recreating the outline in Leo. The
1775 representation in the derived file doesn't matter much.
1776 """
1777 at = self
1778 # Suppress warnings about @file nodes.
1779 at.putOpenNodeSentinel(p, inAtAll=True)
1780 at.putAtAllBody(p)
1781 for child in p.children():
1782 at.putAtAllChild(child) # pragma: no cover (recursive call)
1783 #@+node:ekr.20041005105605.170: *6* at: @others
1784 #@+node:ekr.20041005105605.173: *7* at.putAtOthersLine & helper
1785 def putAtOthersLine(self, s, i, p):
1786 """Put the expansion of @others."""
1787 at = self
1788 j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1789 k = g.skip_to_end_of_line(s, i)
1790 at.putLeadInSentinel(s, i, j)
1791 at.indent += delta
1792 # s[j:k] starts with '@others'
1793 # Never write lws in new sentinels.
1794 at.putSentinel("@+" + s[j + 1 : k].strip())
1795 for child in p.children():
1796 p = child.copy()
1797 after = p.nodeAfterTree()
1798 while p and p != after:
1799 if at.validInAtOthers(p):
1800 at.putOpenNodeSentinel(p)
1801 at_others_flag = at.putBody(p)
1802 if at_others_flag:
1803 p.moveToNodeAfterTree()
1804 else:
1805 p.moveToThreadNext()
1806 else:
1807 p.moveToNodeAfterTree()
1808 # This is the same in both old and new sentinels.
1809 at.putSentinel("@-others")
1810 at.indent -= delta
1811 #@+node:ekr.20041005105605.171: *8* at.validInAtOthers
1812 def validInAtOthers(self, p):
1813 """
1814 Return True if p should be included in the expansion of the @others
1815 directive in the body text of p's parent.
1816 """
1817 at = self
1818 i = g.skip_ws(p.h, 0)
1819 isSection, junk = at.isSectionName(p.h, i)
1820 if isSection:
1821 return False # A section definition node.
1822 if at.sentinels:
1823 # @ignore must not stop expansion here!
1824 return True
1825 if p.isAtIgnoreNode(): # pragma: no cover
1826 g.error('did not write @ignore node', p.v.h)
1827 return False
1828 return True
1829 #@+node:ekr.20041005105605.199: *6* at.findSectionName
1830 def findSectionName(self, s, i, p):
1831 """
1832 Return n1, n2 representing a section name.
1834 Return the reference, *including* brackes.
1835 """
1836 at = self
1838 def is_space(i1, i2):
1839 """A replacement for s[i1 : i2] that doesn't create any substring."""
1840 return i == j or all(s[z] in ' \t\n' for z in range(i1, i2))
1842 end = s.find('\n', i)
1843 j = len(s) if end == -1 else end
1844 # Careful: don't look beyond the end of the line!
1845 if end == -1:
1846 n1 = s.find(at.section_delim1, i)
1847 n2 = s.find(at.section_delim2, i)
1848 else:
1849 n1 = s.find(at.section_delim1, i, end)
1850 n2 = s.find(at.section_delim2, i, end)
1851 n3 = n2 + len(at.section_delim2)
1852 if -1 < n1 < n2: # A *possible* section reference.
1853 if is_space(i, n1) and is_space(n3, j): # A *real* section reference.
1854 return s[n1:n3], n1, n3
1855 # An apparent section reference.
1856 if 'sections' in g.app.debug and not g.unitTesting: # pragma: no cover
1857 i1, i2 = g.getLine(s, i)
1858 g.es_print('Ignoring apparent section reference:', color='red')
1859 g.es_print('Node: ', p.h)
1860 g.es_print('Line: ', s[i1:i2].rstrip())
1861 return None, 0, 0
1862 #@+node:ekr.20041005105605.174: *6* at.putCodeLine
1863 def putCodeLine(self, s, i):
1864 """Put a normal code line."""
1865 at = self
1866 # Put @verbatim sentinel if required.
1867 k = g.skip_ws(s, i)
1868 if g.match(s, k, self.startSentinelComment + '@'):
1869 self.putSentinel('@verbatim')
1870 j = g.skip_line(s, i)
1871 line = s[i:j]
1872 # Don't put any whitespace in otherwise blank lines.
1873 if len(line) > 1: # Preserve *anything* the user puts on the line!!!
1874 at.putIndent(at.indent, line)
1875 if line[-1:] == '\n':
1876 at.os(line[:-1])
1877 at.onl()
1878 else:
1879 at.os(line)
1880 elif line and line[-1] == '\n':
1881 at.onl()
1882 elif line:
1883 at.os(line) # Bug fix: 2013/09/16
1884 else:
1885 g.trace('Can not happen: completely empty line') # pragma: no cover
1886 #@+node:ekr.20041005105605.176: *6* at.putRefLine
1887 def putRefLine(self, s, i, n1, n2, name, p):
1888 """
1889 Put a line containing one or more references.
1891 Important: the so-called name *must* include brackets.
1892 """
1893 at = self
1894 ref = g.findReference(name, p)
1895 if ref:
1896 junk, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1897 at.putLeadInSentinel(s, i, n1)
1898 at.indent += delta
1899 at.putSentinel("@+" + name)
1900 at.putOpenNodeSentinel(ref)
1901 at.putBody(ref)
1902 at.putSentinel("@-" + name)
1903 at.indent -= delta
1904 return
1905 if hasattr(at, 'allow_undefined_refs'): # pragma: no cover
1906 p.v.setVisited() # #2311
1907 # Allow apparent section reference: just write the line.
1908 at.putCodeLine(s, i)
1909 else: # pragma: no cover
1910 # Do give this error even if unit testing.
1911 at.writeError(
1912 f"undefined section: {g.truncate(name, 60)}\n"
1913 f" referenced from: {g.truncate(p.h, 60)}")
1914 #@+node:ekr.20041005105605.180: *5* writing doc lines...
1915 #@+node:ekr.20041005105605.181: *6* at.putBlankDocLine
1916 def putBlankDocLine(self):
1917 at = self
1918 if not at.endSentinelComment:
1919 at.putIndent(at.indent)
1920 at.os(at.startSentinelComment)
1921 # #1496: Retire the @doc convention.
1922 # Remove the blank.
1923 # at.oblank()
1924 at.onl()
1925 #@+node:ekr.20041005105605.183: *6* at.putDocLine
1926 def putDocLine(self, s, i):
1927 """Handle one line of a doc part."""
1928 at = self
1929 j = g.skip_line(s, i)
1930 s = s[i:j]
1931 #
1932 # #1496: Retire the @doc convention:
1933 # Strip all trailing ws here.
1934 if not s.strip():
1935 # A blank line.
1936 at.putBlankDocLine()
1937 return
1938 # Write the line as it is.
1939 at.putIndent(at.indent)
1940 if not at.endSentinelComment:
1941 at.os(at.startSentinelComment)
1942 # #1496: Retire the @doc convention.
1943 # Leave this blank. The line is not blank.
1944 at.oblank()
1945 at.os(s)
1946 if not s.endswith('\n'):
1947 at.onl() # pragma: no cover
1948 #@+node:ekr.20041005105605.185: *6* at.putEndDocLine
1949 def putEndDocLine(self):
1950 """Write the conclusion of a doc part."""
1951 at = self
1952 # Put the closing delimiter if we are using block comments.
1953 if at.endSentinelComment:
1954 at.putIndent(at.indent)
1955 at.os(at.endSentinelComment)
1956 at.onl() # Note: no trailing whitespace.
1957 #@+node:ekr.20041005105605.182: *6* at.putStartDocLine
1958 def putStartDocLine(self, s, i, kind):
1959 """Write the start of a doc part."""
1960 at = self
1961 sentinel = "@+doc" if kind == at.docDirective else "@+at"
1962 directive = "@doc" if kind == at.docDirective else "@"
1963 # Put whatever follows the directive in the sentinel.
1964 # Skip past the directive.
1965 i += len(directive)
1966 j = g.skip_to_end_of_line(s, i)
1967 follow = s[i:j]
1968 # Put the opening @+doc or @-doc sentinel, including whatever follows the directive.
1969 at.putSentinel(sentinel + follow)
1970 # Put the opening comment if we are using block comments.
1971 if at.endSentinelComment:
1972 at.putIndent(at.indent)
1973 at.os(at.startSentinelComment)
1974 at.onl()
1975 #@+node:ekr.20041005105605.187: *4* Writing sentinels...
1976 #@+node:ekr.20041005105605.188: *5* at.nodeSentinelText & helper
1977 def nodeSentinelText(self, p):
1978 """Return the text of a @+node or @-node sentinel for p."""
1979 at = self
1980 h = at.removeCommentDelims(p)
1981 if getattr(at, 'at_shadow_test_hack', False): # pragma: no cover
1982 # A hack for @shadow unit testing.
1983 # see AtShadowTestCase.makePrivateLines.
1984 return h
1985 gnx = p.v.fileIndex
1986 level = 1 + p.level() - self.root.level()
1987 if level > 2:
1988 return f"{gnx}: *{level}* {h}"
1989 return f"{gnx}: {'*' * level} {h}"
1990 #@+node:ekr.20041005105605.189: *6* at.removeCommentDelims
1991 def removeCommentDelims(self, p):
1992 """
1993 If the present @language/@comment settings do not specify a single-line comment
1994 we remove all block comment delims from h. This prevents headline text from
1995 interfering with the parsing of node sentinels.
1996 """
1997 at = self
1998 start = at.startSentinelComment
1999 end = at.endSentinelComment
2000 h = p.h
2001 if end:
2002 h = h.replace(start, "")
2003 h = h.replace(end, "")
2004 return h
2005 #@+node:ekr.20041005105605.190: *5* at.putLeadInSentinel
2006 def putLeadInSentinel(self, s, i, j):
2007 """
2008 Set at.leadingWs as needed for @+others and @+<< sentinels.
2010 i points at the start of a line.
2011 j points at @others or a section reference.
2012 """
2013 at = self
2014 at.leadingWs = "" # Set the default.
2015 if i == j:
2016 return # The @others or ref starts a line.
2017 k = g.skip_ws(s, i)
2018 if j == k:
2019 # Remember the leading whitespace, including its spelling.
2020 at.leadingWs = s[i:j]
2021 else:
2022 self.putIndent(at.indent) # 1/29/04: fix bug reported by Dan Winkler.
2023 at.os(s[i:j])
2024 at.onl_sent()
2025 #@+node:ekr.20041005105605.192: *5* at.putOpenLeoSentinel 4.x
2026 def putOpenLeoSentinel(self, s):
2027 """Write @+leo sentinel."""
2028 at = self
2029 if at.sentinels or hasattr(at, 'force_sentinels'):
2030 s = s + "-thin"
2031 encoding = at.encoding.lower()
2032 if encoding != "utf-8": # pragma: no cover
2033 # New in 4.2: encoding fields end in ",."
2034 s = s + f"-encoding={encoding},."
2035 at.putSentinel(s)
2036 #@+node:ekr.20041005105605.193: *5* at.putOpenNodeSentinel
2037 def putOpenNodeSentinel(self, p, inAtAll=False):
2038 """Write @+node sentinel for p."""
2039 # Note: lineNumbers.py overrides this method.
2040 at = self
2041 if not inAtAll and p.isAtFileNode() and p != at.root: # pragma: no cover
2042 at.writeError("@file not valid in: " + p.h)
2043 return
2044 s = at.nodeSentinelText(p)
2045 at.putSentinel("@+node:" + s)
2046 # Leo 4.7: we never write tnodeLists.
2047 #@+node:ekr.20041005105605.194: *5* at.putSentinel (applies cweb hack) 4.x
2048 def putSentinel(self, s):
2049 """
2050 Write a sentinel whose text is s, applying the CWEB hack if needed.
2052 This method outputs all sentinels.
2053 """
2054 at = self
2055 if at.sentinels or hasattr(at, 'force_sentinels'):
2056 at.putIndent(at.indent)
2057 at.os(at.startSentinelComment)
2058 # #2194. The following would follow the black convention,
2059 # but doing so is a dubious idea.
2060 # at.os(' ')
2061 # Apply the cweb hack to s:
2062 # If the opening comment delim ends in '@',
2063 # double all '@' signs except the first.
2064 start = at.startSentinelComment
2065 if start and start[-1] == '@':
2066 s = s.replace('@', '@@')[1:]
2067 at.os(s)
2068 if at.endSentinelComment:
2069 at.os(at.endSentinelComment)
2070 at.onl()
2071 #@+node:ekr.20041005105605.196: *4* Writing utils...
2072 #@+node:ekr.20181024134823.1: *5* at.addToOrphanList
2073 def addToOrphanList(self, root): # pragma: no cover
2074 """Mark the root as erroneous for c.raise_error_dialogs()."""
2075 c = self.c
2076 # Fix #1050:
2077 root.setOrphan()
2078 c.orphan_at_file_nodes.append(root.h)
2079 #@+node:ekr.20220120210617.1: *5* at.checkPyflakes
2080 def checkPyflakes(self, contents, fileName, root): # pragma: no cover
2081 at = self
2082 ok = True
2083 if g.unitTesting or not at.runPyFlakesOnWrite:
2084 return ok
2085 if not contents or not fileName or not fileName.endswith('.py'):
2086 return ok
2087 ok = self.runPyflakes(root)
2088 if not ok:
2089 g.app.syntax_error_files.append(g.shortFileName(fileName))
2090 return ok
2091 #@+node:ekr.20090514111518.5661: *5* at.checkPythonCode & helpers
2092 def checkPythonCode(self, contents, fileName, root): # pragma: no cover
2093 """Perform python-related checks on root."""
2094 at = self
2095 if g.unitTesting or not contents or not fileName or not fileName.endswith('.py'):
2096 return
2097 ok = True
2098 if at.checkPythonCodeOnWrite:
2099 ok = at.checkPythonSyntax(root, contents)
2100 if ok and at.runPyFlakesOnWrite:
2101 ok = self.runPyflakes(root)
2102 if not ok:
2103 g.app.syntax_error_files.append(g.shortFileName(fileName))
2104 #@+node:ekr.20090514111518.5663: *6* at.checkPythonSyntax
2105 def checkPythonSyntax(self, p, body):
2106 at = self
2107 try:
2108 body = body.replace('\r', '')
2109 fn = f"<node: {p.h}>"
2110 compile(body + '\n', fn, 'exec')
2111 return True
2112 except SyntaxError: # pragma: no cover
2113 if not g.unitTesting:
2114 at.syntaxError(p, body)
2115 except Exception: # pragma: no cover
2116 g.trace("unexpected exception")
2117 g.es_exception()
2118 return False
2119 #@+node:ekr.20090514111518.5666: *7* at.syntaxError (leoAtFile)
2120 def syntaxError(self, p, body): # pragma: no cover
2121 """Report a syntax error."""
2122 g.error(f"Syntax error in: {p.h}")
2123 typ, val, tb = sys.exc_info()
2124 message = hasattr(val, 'message') and val.message
2125 if message:
2126 g.es_print(message)
2127 if val is None:
2128 return
2129 lines = g.splitLines(body)
2130 n = val.lineno
2131 offset = val.offset or 0
2132 if n is None:
2133 return
2134 i = val.lineno - 1
2135 for j in range(max(0, i - 2), min(i + 2, len(lines) - 1)):
2136 line = lines[j].rstrip()
2137 if j == i:
2138 unl = p.get_UNL()
2139 g.es_print(f"{j+1:5}:* {line}", nodeLink=f"{unl}::-{j+1}") # Global line.
2140 g.es_print(' ' * (7 + offset) + '^')
2141 else:
2142 g.es_print(f"{j+1:5}: {line}")
2143 #@+node:ekr.20161021084954.1: *6* at.runPyflakes
2144 def runPyflakes(self, root): # pragma: no cover
2145 """Run pyflakes on the selected node."""
2146 try:
2147 from leo.commands import checkerCommands
2148 if checkerCommands.pyflakes:
2149 x = checkerCommands.PyflakesCommand(self.c)
2150 ok = x.run(root)
2151 return ok
2152 return True # Suppress error if pyflakes can not be imported.
2153 except Exception:
2154 g.es_exception()
2155 return True # Pretend all is well
2156 #@+node:ekr.20041005105605.198: *5* at.directiveKind4 (write logic)
2157 # These patterns exclude constructs such as @encoding.setter or @encoding(whatever)
2158 # However, they must allow @language python, @nocolor-node, etc.
2160 at_directive_kind_pattern = re.compile(r'\s*@([\w-]+)\s*')
2162 def directiveKind4(self, s, i):
2163 """
2164 Return the kind of at-directive or noDirective.
2166 Potential simplifications:
2167 - Using strings instead of constants.
2168 - Using additional regex's to recognize directives.
2169 """
2170 at = self
2171 n = len(s)
2172 if i >= n or s[i] != '@':
2173 j = g.skip_ws(s, i)
2174 if g.match_word(s, j, "@others"):
2175 return at.othersDirective
2176 if g.match_word(s, j, "@all"):
2177 return at.allDirective
2178 return at.noDirective
2179 table = (
2180 ("@all", at.allDirective),
2181 ("@c", at.cDirective),
2182 ("@code", at.codeDirective),
2183 ("@doc", at.docDirective),
2184 ("@others", at.othersDirective),
2185 ("@verbatim", at.startVerbatim))
2186 # ("@end_raw", at.endRawDirective), # #2276.
2187 # ("@raw", at.rawDirective), # #2276
2188 # Rewritten 6/8/2005.
2189 if i + 1 >= n or s[i + 1] in (' ', '\t', '\n'):
2190 # Bare '@' not recognized in cweb mode.
2191 return at.noDirective if at.language == "cweb" else at.atDirective
2192 if not s[i + 1].isalpha():
2193 return at.noDirective # Bug fix: do NOT return miscDirective here!
2194 if at.language == "cweb" and g.match_word(s, i, '@c'):
2195 return at.noDirective
2196 # When the language is elixir, @doc followed by a space and string delimiter
2197 # needs to be treated as plain text; the following does not enforce the
2198 # 'string delimiter' part of that. An @doc followed by something other than
2199 # a space will fall through to usual Leo @doc processing.
2200 if at.language == "elixir" and g.match_word(s, i, '@doc '): # pragma: no cover
2201 return at.noDirective
2202 for name, directive in table:
2203 if g.match_word(s, i, name):
2204 return directive
2205 # Support for add_directives plugin.
2206 # Use regex to properly distinguish between Leo directives
2207 # and python decorators.
2208 s2 = s[i:]
2209 m = self.at_directive_kind_pattern.match(s2)
2210 if m:
2211 word = m.group(1)
2212 if word not in g.globalDirectiveList:
2213 return at.noDirective
2214 s3 = s2[m.end(1) :]
2215 if s3 and s3[0] in ".(":
2216 return at.noDirective
2217 return at.miscDirective
2218 # An unusual case.
2219 return at.noDirective # pragma: no cover
2220 #@+node:ekr.20041005105605.200: *5* at.isSectionName
2221 # returns (flag, end). end is the index of the character after the section name.
2223 def isSectionName(self, s, i): # pragma: no cover
2225 at = self
2226 # Allow leading periods.
2227 while i < len(s) and s[i] == '.':
2228 i += 1
2229 if not g.match(s, i, at.section_delim1):
2230 return False, -1
2231 i = g.find_on_line(s, i, at.section_delim2)
2232 if i > -1:
2233 return True, i + len(at.section_delim2)
2234 return False, -1
2235 #@+node:ekr.20190111112442.1: *5* at.isWritable
2236 def isWritable(self, path): # pragma: no cover
2237 """Return True if the path is writable."""
2238 try:
2239 # os.access() may not exist on all platforms.
2240 ok = os.access(path, os.W_OK)
2241 except AttributeError:
2242 return True
2243 if not ok:
2244 g.es('read only:', repr(path), color='red')
2245 return ok
2246 #@+node:ekr.20041005105605.201: *5* at.os and allies
2247 #@+node:ekr.20041005105605.202: *6* at.oblank, oblanks & otabs
2248 def oblank(self):
2249 self.os(' ')
2251 def oblanks(self, n): # pragma: no cover
2252 self.os(' ' * abs(n))
2254 def otabs(self, n): # pragma: no cover
2255 self.os('\t' * abs(n))
2256 #@+node:ekr.20041005105605.203: *6* at.onl & onl_sent
2257 def onl(self):
2258 """Write a newline to the output stream."""
2259 self.os('\n') # **not** self.output_newline
2261 def onl_sent(self):
2262 """Write a newline to the output stream, provided we are outputting sentinels."""
2263 if self.sentinels:
2264 self.onl()
2265 #@+node:ekr.20041005105605.204: *6* at.os
2266 def os(self, s):
2267 """
2268 Append a string to at.outputList.
2270 All output produced by leoAtFile module goes here.
2271 """
2272 at = self
2273 if s.startswith(self.underindentEscapeString): # pragma: no cover
2274 try:
2275 junk, s = at.parseUnderindentTag(s)
2276 except Exception:
2277 at.exception("exception writing:" + s)
2278 return
2279 s = g.toUnicode(s, at.encoding)
2280 at.outputList.append(s)
2281 #@+node:ekr.20041005105605.205: *5* at.outputStringWithLineEndings
2282 def outputStringWithLineEndings(self, s): # pragma: no cover
2283 """
2284 Write the string s as-is except that we replace '\n' with the proper line ending.
2286 Calling self.onl() runs afoul of queued newlines.
2287 """
2288 at = self
2289 s = g.toUnicode(s, at.encoding)
2290 s = s.replace('\n', at.output_newline)
2291 self.os(s)
2292 #@+node:ekr.20190111045822.1: *5* at.precheck (calls shouldPrompt...)
2293 def precheck(self, fileName, root): # pragma: no cover
2294 """
2295 Check whether a dirty, potentially dangerous, file should be written.
2297 Return True if so. Return False *and* issue a warning otherwise.
2298 """
2299 at = self
2300 #
2301 # #1450: First, check that the directory exists.
2302 theDir = g.os_path_dirname(fileName)
2303 if theDir and not g.os_path_exists(theDir):
2304 at.error(f"Directory not found:\n{theDir}")
2305 return False
2306 #
2307 # Now check the file.
2308 if not at.shouldPromptForDangerousWrite(fileName, root):
2309 # Fix bug 889175: Remember the full fileName.
2310 at.rememberReadPath(fileName, root)
2311 return True
2312 #
2313 # Prompt if the write would overwrite the existing file.
2314 ok = self.promptForDangerousWrite(fileName)
2315 if ok:
2316 # Fix bug 889175: Remember the full fileName.
2317 at.rememberReadPath(fileName, root)
2318 return True
2319 #
2320 # Fix #1031: do not add @ignore here!
2321 g.es("not written:", fileName)
2322 return False
2323 #@+node:ekr.20050506090446.1: *5* at.putAtFirstLines
2324 def putAtFirstLines(self, s):
2325 """
2326 Write any @firstlines from string s.
2327 These lines are converted to @verbatim lines,
2328 so the read logic simply ignores lines preceding the @+leo sentinel.
2329 """
2330 at = self
2331 tag = "@first"
2332 i = 0
2333 while g.match(s, i, tag):
2334 i += len(tag)
2335 i = g.skip_ws(s, i)
2336 j = i
2337 i = g.skip_to_end_of_line(s, i)
2338 # Write @first line, whether empty or not
2339 line = s[j:i]
2340 at.os(line)
2341 at.onl()
2342 i = g.skip_nl(s, i)
2343 #@+node:ekr.20050506090955: *5* at.putAtLastLines
2344 def putAtLastLines(self, s):
2345 """
2346 Write any @last lines from string s.
2347 These lines are converted to @verbatim lines,
2348 so the read logic simply ignores lines following the @-leo sentinel.
2349 """
2350 at = self
2351 tag = "@last"
2352 # Use g.splitLines to preserve trailing newlines.
2353 lines = g.splitLines(s)
2354 n = len(lines)
2355 j = k = n - 1
2356 # Scan backwards for @last directives.
2357 while j >= 0:
2358 line = lines[j]
2359 if g.match(line, 0, tag):
2360 j -= 1
2361 elif not line.strip():
2362 j -= 1
2363 else:
2364 break # pragma: no cover (coverage bug)
2365 # Write the @last lines.
2366 for line in lines[j + 1 : k + 1]:
2367 if g.match(line, 0, tag):
2368 i = len(tag)
2369 i = g.skip_ws(line, i)
2370 at.os(line[i:])
2371 #@+node:ekr.20041005105605.206: *5* at.putDirective & helper
2372 def putDirective(self, s, i, p):
2373 r"""
2374 Output a sentinel a directive or reference s.
2376 It is important for PHP and other situations that \@first and \@last
2377 directives get translated to verbatim lines that do *not* include what
2378 follows the @first & @last directives.
2379 """
2380 at = self
2381 k = i
2382 j = g.skip_to_end_of_line(s, i)
2383 directive = s[i:j]
2384 if g.match_word(s, k, "@delims"):
2385 at.putDelims(directive, s, k)
2386 elif g.match_word(s, k, "@language"):
2387 self.putSentinel("@" + directive)
2388 elif g.match_word(s, k, "@comment"):
2389 self.putSentinel("@" + directive)
2390 elif g.match_word(s, k, "@last"):
2391 # #1307.
2392 if p.isAtCleanNode(): # pragma: no cover
2393 at.error(f"ignoring @last directive in {p.h!r}")
2394 g.es_print('@last is not valid in @clean nodes')
2395 # #1297.
2396 elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
2397 # Convert to an verbatim line _without_ anything else.
2398 self.putSentinel("@@last")
2399 else:
2400 at.error(f"ignoring @last directive in {p.h!r}") # pragma: no cover
2401 elif g.match_word(s, k, "@first"):
2402 # #1307.
2403 if p.isAtCleanNode(): # pragma: no cover
2404 at.error(f"ignoring @first directive in {p.h!r}")
2405 g.es_print('@first is not valid in @clean nodes')
2406 # #1297.
2407 elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
2408 # Convert to an verbatim line _without_ anything else.
2409 self.putSentinel("@@first")
2410 else:
2411 at.error(f"ignoring @first directive in {p.h!r}") # pragma: no cover
2412 else:
2413 self.putSentinel("@" + directive)
2414 i = g.skip_line(s, k)
2415 return i
2416 #@+node:ekr.20041005105605.207: *6* at.putDelims
2417 def putDelims(self, directive, s, k):
2418 """Put an @delims directive."""
2419 at = self
2420 # Put a space to protect the last delim.
2421 at.putSentinel(directive + " ") # 10/23/02: put @delims, not @@delims
2422 # Skip the keyword and whitespace.
2423 j = i = g.skip_ws(s, k + len("@delims"))
2424 # Get the first delim.
2425 while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
2426 i += 1
2427 if j < i:
2428 at.startSentinelComment = s[j:i]
2429 # Get the optional second delim.
2430 j = i = g.skip_ws(s, i)
2431 while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
2432 i += 1
2433 at.endSentinelComment = s[j:i] if j < i else ""
2434 else:
2435 at.writeError("Bad @delims directive") # pragma: no cover
2436 #@+node:ekr.20041005105605.210: *5* at.putIndent
2437 def putIndent(self, n, s=''): # pragma: no cover
2438 """Put tabs and spaces corresponding to n spaces,
2439 assuming that we are at the start of a line.
2441 Remove extra blanks if the line starts with the underindentEscapeString"""
2442 tag = self.underindentEscapeString
2443 if s.startswith(tag):
2444 n2, s2 = self.parseUnderindentTag(s)
2445 if n2 >= n:
2446 return
2447 if n > 0:
2448 n -= n2
2449 else:
2450 n += n2
2451 if n > 0:
2452 w = self.tab_width
2453 if w > 1:
2454 q, r = divmod(n, w)
2455 self.otabs(q)
2456 self.oblanks(r)
2457 else:
2458 self.oblanks(n)
2459 #@+node:ekr.20041005105605.211: *5* at.putInitialComment
2460 def putInitialComment(self): # pragma: no cover
2461 c = self.c
2462 s2 = c.config.output_initial_comment
2463 if s2:
2464 lines = s2.split("\\n")
2465 for line in lines:
2466 line = line.replace("@date", time.asctime())
2467 if line:
2468 self.putSentinel("@comment " + line)
2469 #@+node:ekr.20190111172114.1: *5* at.replaceFile & helpers
2470 def replaceFile(self, contents, encoding, fileName, root, ignoreBlankLines=False):
2471 """
2472 Write or create the given file from the contents.
2473 Return True if the original file was changed.
2474 """
2475 at, c = self, self.c
2476 if root:
2477 root.clearDirty()
2478 #
2479 # Create the timestamp (only for messages).
2480 if c.config.getBool('log-show-save-time', default=False): # pragma: no cover
2481 format = c.config.getString('log-timestamp-format') or "%H:%M:%S"
2482 timestamp = time.strftime(format) + ' '
2483 else:
2484 timestamp = ''
2485 #
2486 # Adjust the contents.
2487 assert isinstance(contents, str), g.callers()
2488 if at.output_newline != '\n': # pragma: no cover
2489 contents = contents.replace('\r', '').replace('\n', at.output_newline)
2490 #
2491 # If file does not exist, create it from the contents.
2492 fileName = g.os_path_realpath(fileName)
2493 sfn = g.shortFileName(fileName)
2494 if not g.os_path_exists(fileName):
2495 ok = g.writeFile(contents, encoding, fileName)
2496 if ok:
2497 c.setFileTimeStamp(fileName)
2498 if not g.unitTesting:
2499 g.es(f"{timestamp}created: {fileName}") # pragma: no cover
2500 if root:
2501 # Fix bug 889175: Remember the full fileName.
2502 at.rememberReadPath(fileName, root)
2503 at.checkPythonCode(contents, fileName, root)
2504 else:
2505 at.addToOrphanList(root) # pragma: no cover
2506 # No original file to change. Return value tested by a unit test.
2507 return False # No change to original file.
2508 #
2509 # Compare the old and new contents.
2510 old_contents = g.readFileIntoUnicodeString(fileName,
2511 encoding=at.encoding, silent=True)
2512 if not old_contents:
2513 old_contents = ''
2514 unchanged = (
2515 contents == old_contents
2516 or (not at.explicitLineEnding and at.compareIgnoringLineEndings(old_contents, contents))
2517 or ignoreBlankLines and at.compareIgnoringBlankLines(old_contents, contents))
2518 if unchanged:
2519 at.unchangedFiles += 1
2520 if not g.unitTesting and c.config.getBool(
2521 'report-unchanged-files', default=True):
2522 g.es(f"{timestamp}unchanged: {sfn}") # pragma: no cover
2523 # Leo 5.6: Check unchanged files.
2524 at.checkPyflakes(contents, fileName, root)
2525 return False # No change to original file.
2526 #
2527 # Warn if we are only adjusting the line endings.
2528 if at.explicitLineEnding: # pragma: no cover
2529 ok = (
2530 at.compareIgnoringLineEndings(old_contents, contents) or
2531 ignoreBlankLines and at.compareIgnoringLineEndings(
2532 old_contents, contents))
2533 if not ok:
2534 g.warning("correcting line endings in:", fileName)
2535 #
2536 # Write a changed file.
2537 ok = g.writeFile(contents, encoding, fileName)
2538 if ok:
2539 c.setFileTimeStamp(fileName)
2540 if not g.unitTesting:
2541 g.es(f"{timestamp}wrote: {sfn}") # pragma: no cover
2542 else: # pragma: no cover
2543 g.error('error writing', sfn)
2544 g.es('not written:', sfn)
2545 at.addToOrphanList(root)
2546 # Check *after* writing the file.
2547 at.checkPythonCode(contents, fileName, root)
2548 return ok
2549 #@+node:ekr.20190114061452.27: *6* at.compareIgnoringBlankLines
2550 def compareIgnoringBlankLines(self, s1, s2): # pragma: no cover
2551 """Compare two strings, ignoring blank lines."""
2552 assert isinstance(s1, str), g.callers()
2553 assert isinstance(s2, str), g.callers()
2554 if s1 == s2:
2555 return True
2556 s1 = g.removeBlankLines(s1)
2557 s2 = g.removeBlankLines(s2)
2558 return s1 == s2
2559 #@+node:ekr.20190114061452.28: *6* at.compareIgnoringLineEndings
2560 def compareIgnoringLineEndings(self, s1, s2): # pragma: no cover
2561 """Compare two strings, ignoring line endings."""
2562 assert isinstance(s1, str), (repr(s1), g.callers())
2563 assert isinstance(s2, str), (repr(s2), g.callers())
2564 if s1 == s2:
2565 return True
2566 # Wrong: equivalent to ignoreBlankLines!
2567 # s1 = s1.replace('\n','').replace('\r','')
2568 # s2 = s2.replace('\n','').replace('\r','')
2569 s1 = s1.replace('\r', '')
2570 s2 = s2.replace('\r', '')
2571 return s1 == s2
2572 #@+node:ekr.20211029052041.1: *5* at.scanRootForSectionDelims
2573 def scanRootForSectionDelims(self, root):
2574 """
2575 Scan root.b for an "@section-delims" directive.
2576 Set section_delim1 and section_delim2 ivars.
2577 """
2578 at = self
2579 # Set defaults.
2580 at.section_delim1 = '<<'
2581 at.section_delim2 = '>>'
2582 # Scan root.b.
2583 lines = []
2584 for s in g.splitLines(root.b):
2585 m = g.g_section_delims_pat.match(s)
2586 if m:
2587 lines.append(s)
2588 at.section_delim1 = m.group(1)
2589 at.section_delim2 = m.group(2)
2590 # Disallow multiple directives.
2591 if len(lines) > 1: # pragma: no cover
2592 at.error(f"Multiple @section-delims directives in {root.h}")
2593 g.es_print('using default delims')
2594 at.section_delim1 = '<<'
2595 at.section_delim2 = '>>'
2596 #@+node:ekr.20090514111518.5665: *5* at.tabNannyNode
2597 def tabNannyNode(self, p, body):
2598 try:
2599 readline = g.ReadLinesClass(body).next
2600 tabnanny.process_tokens(tokenize.generate_tokens(readline))
2601 except IndentationError: # pragma: no cover
2602 if g.unitTesting:
2603 raise
2604 junk2, msg, junk = sys.exc_info()
2605 g.error("IndentationError in", p.h)
2606 g.es('', str(msg))
2607 except tokenize.TokenError: # pragma: no cover
2608 if g.unitTesting:
2609 raise
2610 junk3, msg, junk = sys.exc_info()
2611 g.error("TokenError in", p.h)
2612 g.es('', str(msg))
2613 except tabnanny.NannyNag: # pragma: no cover
2614 if g.unitTesting:
2615 raise
2616 junk4, nag, junk = sys.exc_info()
2617 badline = nag.get_lineno()
2618 line = nag.get_line()
2619 message = nag.get_msg()
2620 g.error("indentation error in", p.h, "line", badline)
2621 g.es(message)
2622 line2 = repr(str(line))[1:-1]
2623 g.es("offending line:\n", line2)
2624 except Exception: # pragma: no cover
2625 g.trace("unexpected exception")
2626 g.es_exception()
2627 raise
2628 #@+node:ekr.20041005105605.216: *5* at.warnAboutOrpanAndIgnoredNodes
2629 # Called from putFile.
2631 def warnAboutOrphandAndIgnoredNodes(self): # pragma: no cover
2632 # Always warn, even when language=="cweb"
2633 at, root = self, self.root
2634 if at.errors:
2635 return # No need to repeat this.
2636 for p in root.self_and_subtree(copy=False):
2637 if not p.v.isVisited():
2638 at.writeError("Orphan node: " + p.h)
2639 if p.hasParent():
2640 g.blue("parent node:", p.parent().h)
2641 p = root.copy()
2642 after = p.nodeAfterTree()
2643 while p and p != after:
2644 if p.isAtAllNode():
2645 p.moveToNodeAfterTree()
2646 else:
2647 # #1050: test orphan bit.
2648 if p.isOrphan():
2649 at.writeError("Orphan node: " + p.h)
2650 if p.hasParent():
2651 g.blue("parent node:", p.parent().h)
2652 p.moveToThreadNext()
2653 #@+node:ekr.20041005105605.217: *5* at.writeError
2654 def writeError(self, message): # pragma: no cover
2655 """Issue an error while writing an @<file> node."""
2656 at = self
2657 if at.errors == 0:
2658 fn = at.targetFileName or 'unnamed file'
2659 g.es_error(f"errors writing: {fn}")
2660 at.error(message)
2661 at.addToOrphanList(at.root)
2662 #@+node:ekr.20041005105605.218: *5* at.writeException
2663 def writeException(self, fileName, root): # pragma: no cover
2664 at = self
2665 g.error("exception writing:", fileName)
2666 g.es_exception()
2667 if getattr(at, 'outputFile', None):
2668 at.outputFile.flush()
2669 at.outputFile.close()
2670 at.outputFile = None
2671 at.remove(fileName)
2672 at.addToOrphanList(root)
2673 #@+node:ekr.20041005105605.219: *3* at.Utilites
2674 #@+node:ekr.20041005105605.220: *4* at.error & printError
2675 def error(self, *args): # pragma: no cover
2676 at = self
2677 at.printError(*args)
2678 at.errors += 1
2680 def printError(self, *args): # pragma: no cover
2681 """Print an error message that may contain non-ascii characters."""
2682 at = self
2683 if at.errors:
2684 g.error(*args)
2685 else:
2686 g.warning(*args)
2687 #@+node:ekr.20041005105605.221: *4* at.exception
2688 def exception(self, message): # pragma: no cover
2689 self.error(message)
2690 g.es_exception()
2691 #@+node:ekr.20050104131929: *4* at.file operations...
2692 # Error checking versions of corresponding functions in Python's os module.
2693 #@+node:ekr.20050104131820: *5* at.chmod
2694 def chmod(self, fileName, mode): # pragma: no cover
2695 # Do _not_ call self.error here.
2696 if mode is None:
2697 return
2698 try:
2699 os.chmod(fileName, mode)
2700 except Exception:
2701 g.es("exception in os.chmod", fileName)
2702 g.es_exception()
2704 #@+node:ekr.20050104132018: *5* at.remove
2705 def remove(self, fileName): # pragma: no cover
2706 if not fileName:
2707 g.trace('No file name', g.callers())
2708 return False
2709 try:
2710 os.remove(fileName)
2711 return True
2712 except Exception:
2713 if not g.unitTesting:
2714 self.error(f"exception removing: {fileName}")
2715 g.es_exception()
2716 return False
2717 #@+node:ekr.20050104132026: *5* at.stat
2718 def stat(self, fileName): # pragma: no cover
2719 """Return the access mode of named file, removing any setuid, setgid, and sticky bits."""
2720 # Do _not_ call self.error here.
2721 try:
2722 mode = (os.stat(fileName))[0] & (7 * 8 * 8 + 7 * 8 + 7) # 0777
2723 except Exception:
2724 mode = None
2725 return mode
2727 #@+node:ekr.20090530055015.6023: *4* at.get/setPathUa
2728 def getPathUa(self, p):
2729 if hasattr(p.v, 'tempAttributes'):
2730 d = p.v.tempAttributes.get('read-path', {})
2731 return d.get('path')
2732 return ''
2734 def setPathUa(self, p, path):
2735 if not hasattr(p.v, 'tempAttributes'):
2736 p.v.tempAttributes = {}
2737 d = p.v.tempAttributes.get('read-path', {})
2738 d['path'] = path
2739 p.v.tempAttributes['read-path'] = d
2740 #@+node:ekr.20081216090156.4: *4* at.parseUnderindentTag
2741 # Important: this is part of the *write* logic.
2742 # It is called from at.os and at.putIndent.
2744 def parseUnderindentTag(self, s): # pragma: no cover
2745 tag = self.underindentEscapeString
2746 s2 = s[len(tag) :]
2747 # To be valid, the escape must be followed by at least one digit.
2748 i = 0
2749 while i < len(s2) and s2[i].isdigit():
2750 i += 1
2751 if i > 0:
2752 n = int(s2[:i])
2753 # Bug fix: 2012/06/05: remove any period following the count.
2754 # This is a new convention.
2755 if i < len(s2) and s2[i] == '.':
2756 i += 1
2757 return n, s2[i:]
2758 return 0, s
2759 #@+node:ekr.20090712050729.6017: *4* at.promptForDangerousWrite
2760 def promptForDangerousWrite(self, fileName, message=None): # pragma: no cover
2761 """Raise a dialog asking the user whether to overwrite an existing file."""
2762 at, c, root = self, self.c, self.root
2763 if at.cancelFlag:
2764 assert at.canCancelFlag
2765 return False
2766 if at.yesToAll:
2767 assert at.canCancelFlag
2768 return True
2769 if root and root.h.startswith('@auto-rst'):
2770 # Fix bug 50: body text lost switching @file to @auto-rst
2771 # Refuse to convert any @<file> node to @auto-rst.
2772 d = root.v.at_read if hasattr(root.v, 'at_read') else {}
2773 aList = sorted(d.get(fileName, []))
2774 for h in aList:
2775 if not h.startswith('@auto-rst'):
2776 g.es('can not convert @file to @auto-rst!', color='red')
2777 g.es('reverting to:', h)
2778 root.h = h
2779 c.redraw()
2780 return False
2781 if message is None:
2782 message = (
2783 f"{g.splitLongFileName(fileName)}\n"
2784 f"{g.tr('already exists.')}\n"
2785 f"{g.tr('Overwrite this file?')}")
2786 result = g.app.gui.runAskYesNoCancelDialog(c,
2787 title='Overwrite existing file?',
2788 yesToAllMessage="Yes To &All",
2789 message=message,
2790 cancelMessage="&Cancel (No To All)",
2791 )
2792 if at.canCancelFlag:
2793 # We are in the writeAll logic so these flags can be set.
2794 if result == 'cancel':
2795 at.cancelFlag = True
2796 elif result == 'yes-to-all':
2797 at.yesToAll = True
2798 return result in ('yes', 'yes-to-all')
2799 #@+node:ekr.20120112084820.10001: *4* at.rememberReadPath
2800 def rememberReadPath(self, fn, p):
2801 """
2802 Remember the files that have been read *and*
2803 the full headline (@<file> type) that caused the read.
2804 """
2805 v = p.v
2806 # Fix bug #50: body text lost switching @file to @auto-rst
2807 if not hasattr(v, 'at_read'):
2808 v.at_read = {} # pragma: no cover
2809 d = v.at_read
2810 aSet = d.get(fn, set())
2811 aSet.add(p.h)
2812 d[fn] = aSet
2813 #@+node:ekr.20080923070954.4: *4* at.scanAllDirectives
2814 def scanAllDirectives(self, p):
2815 """
2816 Scan p and p's ancestors looking for directives,
2817 setting corresponding AtFile ivars.
2818 """
2819 at, c = self, self.c
2820 d = c.scanAllDirectives(p)
2821 #
2822 # Language & delims: Tricky.
2823 lang_dict = d.get('lang-dict') or {}
2824 delims, language = None, None
2825 if lang_dict:
2826 # There was an @delims or @language directive.
2827 language = lang_dict.get('language')
2828 delims = lang_dict.get('delims')
2829 if not language:
2830 # No language directive. Look for @<file> nodes.
2831 # Do *not* used.get('language')!
2832 language = g.getLanguageFromAncestorAtFileNode(p) or 'python'
2833 at.language = language
2834 if not delims:
2835 delims = g.set_delims_from_language(language)
2836 #
2837 # Previously, setting delims was sometimes skipped, depending on kwargs.
2838 #@+<< Set comment strings from delims >>
2839 #@+node:ekr.20080923070954.13: *5* << Set comment strings from delims >> (at.scanAllDirectives)
2840 delim1, delim2, delim3 = delims
2841 # Use single-line comments if we have a choice.
2842 # delim1,delim2,delim3 now correspond to line,start,end
2843 if delim1:
2844 at.startSentinelComment = delim1
2845 at.endSentinelComment = "" # Must not be None.
2846 elif delim2 and delim3:
2847 at.startSentinelComment = delim2
2848 at.endSentinelComment = delim3
2849 else: # pragma: no cover
2850 #
2851 # Emergency!
2852 #
2853 # Issue an error only if at.language has been set.
2854 # This suppresses a message from the markdown importer.
2855 if not g.unitTesting and at.language:
2856 g.trace(repr(at.language), g.callers())
2857 g.es_print("unknown language: using Python comment delimiters")
2858 g.es_print("c.target_language:", c.target_language)
2859 at.startSentinelComment = "#" # This should never happen!
2860 at.endSentinelComment = ""
2861 #@-<< Set comment strings from delims >>
2862 #
2863 # Easy cases
2864 at.encoding = d.get('encoding') or c.config.default_derived_file_encoding
2865 lineending = d.get('lineending')
2866 at.explicitLineEnding = bool(lineending)
2867 at.output_newline = lineending or g.getOutputNewline(c=c)
2868 at.page_width = d.get('pagewidth') or c.page_width
2869 at.tab_width = d.get('tabwidth') or c.tab_width
2870 return {
2871 "encoding": at.encoding,
2872 "language": at.language,
2873 "lineending": at.output_newline,
2874 "pagewidth": at.page_width,
2875 "path": d.get('path'),
2876 "tabwidth": at.tab_width,
2877 }
2878 #@+node:ekr.20120110174009.9965: *4* at.shouldPromptForDangerousWrite
2879 def shouldPromptForDangerousWrite(self, fn, p): # pragma: no cover
2880 """
2881 Return True if Leo should warn the user that p is an @<file> node that
2882 was not read during startup. Writing that file might cause data loss.
2884 See #50: https://github.com/leo-editor/leo-editor/issues/50
2885 """
2886 trace = 'save' in g.app.debug
2887 sfn = g.shortFileName(fn)
2888 c = self.c
2889 efc = g.app.externalFilesController
2890 if p.isAtNoSentFileNode():
2891 # #1450.
2892 # No danger of overwriting a file.
2893 # It was never read.
2894 return False
2895 if not g.os_path_exists(fn):
2896 # No danger of overwriting fn.
2897 if trace:
2898 g.trace('Return False: does not exist:', sfn)
2899 return False
2900 # #1347: Prompt if the external file is newer.
2901 if efc:
2902 # Like c.checkFileTimeStamp.
2903 if c.sqlite_connection and c.mFileName == fn:
2904 # sqlite database file is never actually overwriten by Leo,
2905 # so do *not* check its timestamp.
2906 pass
2907 elif efc.has_changed(fn):
2908 if trace:
2909 g.trace('Return True: changed:', sfn)
2910 return True
2911 if hasattr(p.v, 'at_read'):
2912 # Fix bug #50: body text lost switching @file to @auto-rst
2913 d = p.v.at_read
2914 for k in d:
2915 # Fix bug # #1469: make sure k still exists.
2916 if (
2917 os.path.exists(k) and os.path.samefile(k, fn)
2918 and p.h in d.get(k, set())
2919 ):
2920 d[fn] = d[k]
2921 if trace:
2922 g.trace('Return False: in p.v.at_read:', sfn)
2923 return False
2924 aSet = d.get(fn, set())
2925 if trace:
2926 g.trace(f"Return {p.h not in aSet()}: p.h not in aSet(): {sfn}")
2927 return p.h not in aSet
2928 if trace:
2929 g.trace('Return True: never read:', sfn)
2930 return True # The file was never read.
2931 #@+node:ekr.20041005105605.20: *4* at.warnOnReadOnlyFile
2932 def warnOnReadOnlyFile(self, fn):
2933 # os.access() may not exist on all platforms.
2934 try:
2935 read_only = not os.access(fn, os.W_OK)
2936 except AttributeError: # pragma: no cover
2937 read_only = False
2938 if read_only:
2939 g.error("read only:", fn) # pragma: no cover
2940 #@-others
2941atFile = AtFile # compatibility
2942#@+node:ekr.20180602102448.1: ** class FastAtRead
2943class FastAtRead:
2944 """
2945 Read an exteral file, created from an @file tree.
2946 This is Vitalije's code, edited by EKR.
2947 """
2949 #@+others
2950 #@+node:ekr.20211030193146.1: *3* fast_at.__init__
2951 def __init__(self, c, gnx2vnode):
2953 self.c = c
2954 assert gnx2vnode is not None
2955 self.gnx2vnode = gnx2vnode # The global fc.gnxDict. Keys are gnx's, values are vnodes.
2956 self.path = None
2957 self.root = None
2958 # compiled patterns...
2959 self.after_pat = None
2960 self.all_pat = None
2961 self.code_pat = None
2962 self.comment_pat = None
2963 self.delims_pat = None
2964 self.doc_pat = None
2965 self.first_pat = None
2966 self.last_pat = None
2967 self.node_start_pat = None
2968 self.others_pat = None
2969 self.ref_pat = None
2970 self.section_delims_pat = None
2971 #@+node:ekr.20180602103135.3: *3* fast_at.get_patterns
2972 #@@nobeautify
2974 def get_patterns(self, comment_delims):
2975 """Create regex patterns for the given comment delims."""
2976 # This must be a function, because of @comments & @delims.
2977 comment_delim_start, comment_delim_end = comment_delims
2978 delim1 = re.escape(comment_delim_start)
2979 delim2 = re.escape(comment_delim_end or '')
2980 ref = g.angleBrackets(r'(.*)')
2981 table = (
2982 # These patterns must be mutually exclusive.
2983 ('after', fr'^\s*{delim1}@afterref{delim2}$'), # @afterref
2984 ('all', fr'^(\s*){delim1}@(\+|-)all\b(.*){delim2}$'), # @all
2985 ('code', fr'^\s*{delim1}@@c(ode)?{delim2}$'), # @c and @code
2986 ('comment', fr'^\s*{delim1}@@comment(.*){delim2}'), # @comment
2987 ('delims', fr'^\s*{delim1}@delims(.*){delim2}'), # @delims
2988 ('doc', fr'^\s*{delim1}@\+(at|doc)?(\s.*?)?{delim2}\n'), # @doc or @
2989 ('first', fr'^\s*{delim1}@@first{delim2}$'), # @first
2990 ('last', fr'^\s*{delim1}@@last{delim2}$'), # @last
2991 # @node
2992 ('node_start', fr'^(\s*){delim1}@\+node:([^:]+): \*(\d+)?(\*?) (.*){delim2}$'),
2993 ('others', fr'^(\s*){delim1}@(\+|-)others\b(.*){delim2}$'), # @others
2994 ('ref', fr'^(\s*){delim1}@(\+|-){ref}\s*{delim2}$'), # section ref
2995 # @section-delims
2996 ('section_delims', fr'^\s*{delim1}@@section-delims[ \t]+([^ \w\n\t]+)[ \t]+([^ \w\n\t]+)[ \t]*{delim2}$'),
2997 )
2998 # Set the ivars.
2999 for (name, pattern) in table:
3000 ivar = f"{name}_pat"
3001 assert hasattr(self, ivar), ivar
3002 setattr(self, ivar, re.compile(pattern))
3003 #@+node:ekr.20180602103135.2: *3* fast_at.scan_header
3004 header_pattern = re.compile(
3005 r'''
3006 ^(.+)@\+leo
3007 (-ver=(\d+))?
3008 (-thin)?
3009 (-encoding=(.*)(\.))?
3010 (.*)$''',
3011 re.VERBOSE,
3012 )
3014 def scan_header(self, lines):
3015 """
3016 Scan for the header line, which follows any @first lines.
3017 Return (delims, first_lines, i+1) or None
3018 """
3019 first_lines: List[str] = []
3020 i = 0 # To keep some versions of pylint happy.
3021 for i, line in enumerate(lines):
3022 m = self.header_pattern.match(line)
3023 if m:
3024 delims = m.group(1), m.group(8) or ''
3025 return delims, first_lines, i + 1
3026 first_lines.append(line)
3027 return None # pragma: no cover (defensive)
3028 #@+node:ekr.20180602103135.8: *3* fast_at.scan_lines
3029 def scan_lines(self, comment_delims, first_lines, lines, path, start):
3030 """Scan all lines of the file, creating vnodes."""
3031 #@+<< init scan_lines >>
3032 #@+node:ekr.20180602103135.9: *4* << init scan_lines >>
3033 #
3034 # Simple vars...
3035 afterref = False # True: the next line follows @afterref.
3036 clone_v = None # The root of the clone tree.
3037 comment_delim1, comment_delim2 = comment_delims # The start/end *comment* delims.
3038 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n') # To handle doc parts.
3039 first_i = 0 # Index into first array.
3040 in_doc = False # True: in @doc parts.
3041 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>' # True: cweb hack in effect.
3042 indent = 0 # The current indentation.
3043 level_stack = [] # Entries are (vnode, in_clone_tree)
3044 n_last_lines = 0 # The number of @@last directives seen.
3045 root_gnx_adjusted = False # True: suppress final checks.
3046 # #1065 so reads will not create spurious child nodes.
3047 root_seen = False # False: The next +@node sentinel denotes the root, regardless of gnx.
3048 section_delim1 = '<<'
3049 section_delim2 = '>>'
3050 section_reference_seen = False
3051 sentinel = comment_delim1 + '@' # Faster than a regex!
3052 # The stack is updated when at+others, at+<section>, or at+all is seen.
3053 stack = [] # Entries are (gnx, indent, body)
3054 # The spelling of at-verbatim sentinel
3055 verbatim_line = comment_delim1 + '@verbatim' + comment_delim2 + '\n'
3056 verbatim = False # True: the next line must be added without change.
3057 #
3058 # Init the parent vnode.
3059 #
3060 root_gnx = gnx = self.root.gnx
3061 context = self.c
3062 parent_v = self.root.v
3063 root_v = parent_v # Does not change.
3064 level_stack.append((root_v, False),)
3065 #
3066 # Init the gnx dict last.
3067 #
3068 gnx2vnode = self.gnx2vnode # Keys are gnx's, values are vnodes.
3069 gnx2body = {} # Keys are gnxs, values are list of body lines.
3070 gnx2vnode[gnx] = parent_v # Add gnx to the keys
3071 # Add gnx to the keys.
3072 # Body is the list of lines presently being accumulated.
3073 gnx2body[gnx] = body = first_lines
3074 #
3075 # Set the patterns
3076 self.get_patterns(comment_delims)
3077 #@-<< init scan_lines >>
3078 i = 0 # To keep pylint happy.
3079 for i, line in enumerate(lines[start:]):
3080 # Strip the line only once.
3081 strip_line = line.strip()
3082 if afterref:
3083 #@+<< handle afterref line>>
3084 #@+node:ekr.20211102052251.1: *4* << handle afterref line >>
3085 if body: # a List of lines.
3086 body[-1] = body[-1].rstrip() + line
3087 else:
3088 body = [line] # pragma: no cover
3089 afterref = False
3090 #@-<< handle afterref line>>
3091 continue
3092 if verbatim:
3093 #@+<< handle verbatim line >>
3094 #@+node:ekr.20211102052518.1: *4* << handle verbatim line >>
3095 # Previous line was verbatim *sentinel*. Append this line as it is.
3096 body.append(line)
3097 verbatim = False
3098 #@-<< handle verbatim line >>
3099 continue
3100 if line == verbatim_line: # <delim>@verbatim.
3101 verbatim = True
3102 continue
3103 #@+<< finalize line >>
3104 #@+node:ekr.20180602103135.10: *4* << finalize line >>
3105 # Undo the cweb hack.
3106 if is_cweb and line.startswith(sentinel):
3107 line = line[: len(sentinel)] + line[len(sentinel) :].replace('@@', '@')
3108 # Adjust indentation.
3109 if indent and line[:indent].isspace() and len(line) > indent:
3110 line = line[indent:]
3111 #@-<< finalize line >>
3112 if not in_doc and not strip_line.startswith(sentinel): # Faster than a regex!
3113 body.append(line)
3114 continue
3115 # These three sections might clear in_doc.
3116 #@+<< handle @others >>
3117 #@+node:ekr.20180602103135.14: *4* << handle @others >>
3118 m = self.others_pat.match(line)
3119 if m:
3120 in_doc = False
3121 if m.group(2) == '+': # opening sentinel
3122 body.append(f"{m.group(1)}@others{m.group(3) or ''}\n")
3123 stack.append((gnx, indent, body))
3124 indent += m.end(1) # adjust current identation
3125 else: # closing sentinel.
3126 # m.group(2) is '-' because the pattern matched.
3127 try:
3128 gnx, indent, body = stack.pop()
3129 except IndexError:
3130 # #2624: This can happen during git-diff.
3131 pass
3132 continue
3133 #@-<< handle @others >>
3134 #@+<< handle section refs >>
3135 #@+node:ekr.20180602103135.18: *4* << handle section refs >>
3136 # Note: scan_header sets *comment* delims, not *section* delims.
3137 # This section coordinates with the section that handles @section-delims.
3138 m = self.ref_pat.match(line)
3139 if m:
3140 in_doc = False
3141 if m.group(2) == '+':
3142 # Any later @section-delims directive is a serious error.
3143 # This kind of error should have been caught by Leo's atFile write logic.
3144 section_reference_seen = True
3145 # open sentinel.
3146 body.append(m.group(1) + section_delim1 + m.group(3) + section_delim2 + '\n')
3147 stack.append((gnx, indent, body))
3148 indent += m.end(1)
3149 elif stack:
3150 # m.group(2) is '-' because the pattern matched.
3151 gnx, indent, body = stack.pop() # #1232: Only if the stack exists.
3152 continue # 2021/10/29: *always* continue.
3153 #@-<< handle section refs >>
3154 #@+<< handle node_start >>
3155 #@+node:ekr.20180602103135.19: *4* << handle node_start >>
3156 m = self.node_start_pat.match(line)
3157 if m:
3158 in_doc = False
3159 gnx, head = m.group(2), m.group(5)
3160 # m.group(3) is the level number, m.group(4) is the number of stars.
3161 level = int(m.group(3)) if m.group(3) else 1 + len(m.group(4))
3162 v = gnx2vnode.get(gnx)
3163 #
3164 # Case 1: The root @file node. Don't change the headline.
3165 if not root_seen and not v and not g.unitTesting:
3166 # Don't warn about a gnx mismatch in the root.
3167 root_gnx_adjusted = True # pragma: no cover
3168 if not root_seen:
3169 # Fix #1064: The node represents the root, regardless of the gnx!
3170 root_seen = True
3171 clone_v = None
3172 gnx2body[gnx] = body = []
3173 # This case can happen, but not in unit tests.
3174 if not v: # pragma: no cover
3175 # Fix #1064.
3176 v = root_v
3177 # This message is annoying when using git-diff.
3178 # if gnx != root_gnx:
3179 # g.es_print("using gnx from external file: %s" % (v.h), color='blue')
3180 gnx2vnode[gnx] = v
3181 v.fileIndex = gnx
3182 v.children = []
3183 continue
3184 #
3185 # Case 2: We are scanning the descendants of a clone.
3186 parent_v, clone_v = level_stack[level - 2]
3187 if v and clone_v:
3188 # The last version of the body and headline wins..
3189 gnx2body[gnx] = body = []
3190 v._headString = head
3191 # Update the level_stack.
3192 level_stack = level_stack[: level - 1]
3193 level_stack.append((v, clone_v),)
3194 # Always clear the children!
3195 v.children = []
3196 parent_v.children.append(v)
3197 continue
3198 #
3199 # Case 3: we are not already scanning the descendants of a clone.
3200 if v:
3201 # The *start* of a clone tree. Reset the children.
3202 clone_v = v
3203 v.children = []
3204 else:
3205 # Make a new vnode.
3206 v = leoNodes.VNode(context=context, gnx=gnx)
3207 #
3208 # The last version of the body and headline wins.
3209 gnx2vnode[gnx] = v
3210 gnx2body[gnx] = body = []
3211 v._headString = head
3212 #
3213 # Update the stack.
3214 level_stack = level_stack[: level - 1]
3215 level_stack.append((v, clone_v),)
3216 #
3217 # Update the links.
3218 assert v != root_v
3219 parent_v.children.append(v)
3220 v.parents.append(parent_v)
3221 continue
3222 #@-<< handle node_start >>
3223 if in_doc:
3224 #@+<< handle @c or @code >>
3225 #@+node:ekr.20211031033532.1: *4* << handle @c or @code >>
3226 # When delim_end exists the doc block:
3227 # - begins with the opening delim, alone on its own line
3228 # - ends with the closing delim, alone on its own line.
3229 # Both of these lines should be skipped.
3230 #
3231 # #1496: Retire the @doc convention.
3232 # An empty line is no longer a sentinel.
3233 if comment_delim2 and line in doc_skip:
3234 # doc_skip is (comment_delim1 + '\n', delim_end + '\n')
3235 continue
3236 #
3237 # Check for @c or @code.
3238 m = self.code_pat.match(line)
3239 if m:
3240 in_doc = False
3241 body.append('@code\n' if m.group(1) else '@c\n')
3242 continue
3243 #@-<< handle @c or @code >>
3244 else:
3245 #@+<< handle @ or @doc >>
3246 #@+node:ekr.20211031033754.1: *4* << handle @ or @doc >>
3247 m = self.doc_pat.match(line)
3248 if m:
3249 # @+at or @+doc?
3250 doc = '@doc' if m.group(1) == 'doc' else '@'
3251 doc2 = m.group(2) or '' # Trailing text.
3252 if doc2:
3253 body.append(f"{doc}{doc2}\n")
3254 else:
3255 body.append(doc + '\n')
3256 # Enter @doc mode.
3257 in_doc = True
3258 continue
3259 #@-<< handle @ or @doc >>
3260 if line.startswith(comment_delim1 + '@-leo'): # Faster than a regex!
3261 # The @-leo sentinel adds *nothing* to the text.
3262 i += 1
3263 break
3264 # Order doesn't matter.
3265 #@+<< handle @all >>
3266 #@+node:ekr.20180602103135.13: *4* << handle @all >>
3267 m = self.all_pat.match(line)
3268 if m:
3269 # @all tells Leo's *write* code not to check for undefined sections.
3270 # Here, in the read code, we merely need to add it to the body.
3271 # Pushing and popping the stack may not be necessary, but it can't hurt.
3272 if m.group(2) == '+': # opening sentinel
3273 body.append(f"{m.group(1)}@all{m.group(3) or ''}\n")
3274 stack.append((gnx, indent, body))
3275 else: # closing sentinel.
3276 # m.group(2) is '-' because the pattern matched.
3277 gnx, indent, body = stack.pop()
3278 gnx2body[gnx] = body
3279 continue
3280 #@-<< handle @all >>
3281 #@+<< handle afterref >>
3282 #@+node:ekr.20180603063102.1: *4* << handle afterref >>
3283 m = self.after_pat.match(line)
3284 if m:
3285 afterref = True
3286 continue
3287 #@-<< handle afterref >>
3288 #@+<< handle @first and @last >>
3289 #@+node:ekr.20180606053919.1: *4* << handle @first and @last >>
3290 m = self.first_pat.match(line)
3291 if m:
3292 # pylint: disable=no-else-continue
3293 if 0 <= first_i < len(first_lines):
3294 body.append('@first ' + first_lines[first_i])
3295 first_i += 1
3296 continue
3297 else: # pragma: no cover
3298 g.trace(f"\ntoo many @first lines: {path}")
3299 print('@first is valid only at the start of @<file> nodes\n')
3300 g.printObj(first_lines, tag='first_lines')
3301 g.printObj(lines[start : i + 2], tag='lines[start:i+2]')
3302 continue
3303 m = self.last_pat.match(line)
3304 if m:
3305 # Just increment the count of the expected last lines.
3306 # We'll fill in the @last line directives after we see the @-leo directive.
3307 n_last_lines += 1
3308 continue
3309 #@-<< handle @first and @last >>
3310 #@+<< handle @comment >>
3311 #@+node:ekr.20180621050901.1: *4* << handle @comment >>
3312 # http://leoeditor.com/directives.html#part-4-dangerous-directives
3313 m = self.comment_pat.match(line)
3314 if m:
3315 # <1, 2 or 3 comment delims>
3316 delims = m.group(1).strip()
3317 # Whatever happens, retain the @delims line.
3318 body.append(f"@comment {delims}\n")
3319 delim1, delim2, delim3 = g.set_delims_from_string(delims)
3320 # delim1 is always the single-line delimiter.
3321 if delim1:
3322 comment_delim1, comment_delim2 = delim1, ''
3323 else:
3324 comment_delim1, comment_delim2 = delim2, delim3
3325 #
3326 # Within these delimiters:
3327 # - double underscores represent a newline.
3328 # - underscores represent a significant space,
3329 comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
3330 comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')
3331 # Recalculate all delim-related values
3332 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
3333 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
3334 sentinel = comment_delim1 + '@'
3335 #
3336 # Recalculate the patterns.
3337 comment_delims = comment_delim1, comment_delim2
3338 self.get_patterns(comment_delims)
3339 continue
3340 #@-<< handle @comment >>
3341 #@+<< handle @delims >>
3342 #@+node:ekr.20180608104836.1: *4* << handle @delims >>
3343 m = self.delims_pat.match(line)
3344 if m:
3345 # Get 1 or 2 comment delims
3346 # Whatever happens, retain the original @delims line.
3347 delims = m.group(1).strip()
3348 body.append(f"@delims {delims}\n")
3349 #
3350 # Parse the delims.
3351 self.delims_pat = re.compile(r'^([^ ]+)\s*([^ ]+)?')
3352 m2 = self.delims_pat.match(delims)
3353 if not m2: # pragma: no cover
3354 g.trace(f"Ignoring invalid @delims: {line!r}")
3355 continue
3356 comment_delim1 = m2.group(1)
3357 comment_delim2 = m2.group(2) or ''
3358 #
3359 # Within these delimiters:
3360 # - double underscores represent a newline.
3361 # - underscores represent a significant space,
3362 comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
3363 comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')
3364 # Recalculate all delim-related values
3365 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
3366 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
3367 sentinel = comment_delim1 + '@'
3368 #
3369 # Recalculate the patterns
3370 comment_delims = comment_delim1, comment_delim2
3371 self.get_patterns(comment_delims)
3372 continue
3373 #@-<< handle @delims >>
3374 #@+<< handle @section-delims >>
3375 #@+node:ekr.20211030033211.1: *4* << handle @section-delims >>
3376 m = self.section_delims_pat.match(line)
3377 if m:
3378 if section_reference_seen: # pragma: no cover
3379 # This is a serious error.
3380 # This kind of error should have been caught by Leo's atFile write logic.
3381 g.es_print('section-delims seen after a section reference', color='red')
3382 else:
3383 # Carefully update the section reference pattern!
3384 section_delim1 = d1 = re.escape(m.group(1))
3385 section_delim2 = d2 = re.escape(m.group(2) or '')
3386 self.ref_pat = re.compile(fr'^(\s*){comment_delim1}@(\+|-){d1}(.*){d2}\s*{comment_delim2}$')
3387 body.append(f"@section-delims {m.group(1)} {m.group(2)}\n")
3388 continue
3389 #@-<< handle @section-delims >>
3390 # These sections must be last, in this order.
3391 #@+<< handle remaining @@ lines >>
3392 #@+node:ekr.20180603135602.1: *4* << handle remaining @@ lines >>
3393 # @first, @last, @delims and @comment generate @@ sentinels,
3394 # So this must follow all of those.
3395 if line.startswith(comment_delim1 + '@@'):
3396 ii = len(comment_delim1) + 1 # on second '@'
3397 jj = line.rfind(comment_delim2) if comment_delim2 else -1
3398 body.append(line[ii:jj] + '\n')
3399 continue
3400 #@-<< handle remaining @@ lines >>
3401 if in_doc:
3402 #@+<< handle remaining @doc lines >>
3403 #@+node:ekr.20180606054325.1: *4* << handle remaining @doc lines >>
3404 if comment_delim2:
3405 # doc lines are unchanged.
3406 body.append(line)
3407 continue
3408 # Doc lines start with start_delim + one blank.
3409 # #1496: Retire the @doc convention.
3410 # #2194: Strip lws.
3411 tail = line.lstrip()[len(comment_delim1) + 1 :]
3412 if tail.strip():
3413 body.append(tail)
3414 else:
3415 body.append('\n')
3416 continue
3417 #@-<< handle remaining @doc lines >>
3418 #@+<< handle remaining @ lines >>
3419 #@+node:ekr.20180602103135.17: *4* << handle remaining @ lines >>
3420 # Handle an apparent sentinel line.
3421 # This *can* happen after the git-diff or refresh-from-disk commands.
3422 #
3423 if 1: # pragma: no cover (defensive)
3424 # This assert verifies the short-circuit test.
3425 assert strip_line.startswith(sentinel), (repr(sentinel), repr(line))
3426 # A useful trace.
3427 g.trace(
3428 f"{g.shortFileName(self.path)}: "
3429 f"warning: inserting unexpected line: {line.rstrip()!r}"
3430 )
3431 # #2213: *Do* insert the line, with a warning.
3432 body.append(line)
3433 #@-<< handle remaining @ lines >>
3434 else:
3435 # No @-leo sentinel!
3436 return # pragma: no cover
3437 #@+<< final checks >>
3438 #@+node:ekr.20211104054823.1: *4* << final checks >>
3439 if g.unitTesting:
3440 # Unit tests must use the proper value for root.gnx.
3441 assert not root_gnx_adjusted
3442 assert not stack, stack
3443 assert root_gnx == gnx, (root_gnx, gnx)
3444 elif root_gnx_adjusted: # pragma: no cover
3445 pass # Don't check!
3446 elif stack: # pragma: no cover
3447 g.error('scan_lines: Stack should be empty')
3448 g.printObj(stack, tag='stack')
3449 elif root_gnx != gnx: # pragma: no cover
3450 g.error('scan_lines: gnx error')
3451 g.es_print(f"root_gnx: {root_gnx} != gnx: {gnx}")
3452 #@-<< final checks >>
3453 #@+<< insert @last lines >>
3454 #@+node:ekr.20211103101453.1: *4* << insert @last lines >>
3455 tail_lines = lines[start + i :]
3456 if tail_lines:
3457 # Convert the trailing lines to @last directives.
3458 last_lines = [f"@last {z.rstrip()}\n" for z in tail_lines]
3459 # Add the lines to the dictionary of lines.
3460 gnx2body[gnx] = gnx2body[gnx] + last_lines
3461 # Warn if there is an unexpected number of last lines.
3462 if n_last_lines != len(last_lines): # pragma: no cover
3463 n1 = n_last_lines
3464 n2 = len(last_lines)
3465 g.trace(f"Expected {n1} trailing line{g.plural(n1)}, got {n2}")
3466 #@-<< insert @last lines >>
3467 #@+<< post pass: set all body text>>
3468 #@+node:ekr.20211104054426.1: *4* << post pass: set all body text>>
3469 # Set the body text.
3470 assert root_v.gnx in gnx2vnode, root_v
3471 assert root_v.gnx in gnx2body, root_v
3472 for key in gnx2body:
3473 body = gnx2body.get(key)
3474 v = gnx2vnode.get(key)
3475 assert v, (key, v)
3476 v._bodyString = g.toUnicode(''.join(body))
3477 #@-<< post pass: set all body text>>
3478 #@+node:ekr.20180603170614.1: *3* fast_at.read_into_root
3479 def read_into_root(self, contents, path, root):
3480 """
3481 Parse the file's contents, creating a tree of vnodes
3482 anchored in root.v.
3483 """
3484 self.path = path
3485 self.root = root
3486 sfn = g.shortFileName(path)
3487 contents = contents.replace('\r', '')
3488 lines = g.splitLines(contents)
3489 data = self.scan_header(lines)
3490 if not data: # pragma: no cover
3491 g.trace(f"Invalid external file: {sfn}")
3492 return False
3493 # Clear all children.
3494 # Previously, this had been done in readOpenFile.
3495 root.v._deleteAllChildren()
3496 comment_delims, first_lines, start_i = data
3497 self.scan_lines(comment_delims, first_lines, lines, path, start_i)
3498 return True
3499 #@-others
3500#@-others
3501#@@language python
3502#@@tabwidth -4
3503#@@pagewidth 60
3505#@-leo