Coverage for C:\Repos\leo-editor\leo\core\leoShadow.py: 66%
336 statements
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
« prev ^ index » next coverage.py v6.4, created at 2022-05-24 10:21 -0500
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20080708094444.1: * @file leoShadow.py
4#@@first
5#@+<< docstring >>
6#@+node:ekr.20080708094444.78: ** << docstring >>
7"""
8leoShadow.py
10This code allows users to use Leo with files which contain no sentinels
11and still have information flow in both directions between outlines and
12derived files.
14Private files contain sentinels: they live in the Leo-shadow subdirectory.
15Public files contain no sentinels: they live in the parent (main) directory.
17When Leo first reads an @shadow we create a file without sentinels in the regular directory.
19The slightly hard thing to do is to pick up changes from the file without
20sentinels, and put them into the file with sentinels.
22Settings:
23- @string shadow_subdir (default: .leo_shadow): name of the shadow directory.
25- @string shadow_prefix (default: x): prefix of shadow files.
26 This prefix allows the shadow file and the original file to have different names.
27 This is useful for name-based tools like py.test.
28"""
29#@-<< docstring >>
30#@+<< imports >>
31#@+node:ekr.20080708094444.52: ** << imports >> (leoShadow)
32import difflib
33import os
34import pprint
35from typing import List
36from leo.core import leoGlobals as g
37#@-<< imports >>
38#@+others
39#@+node:ekr.20080708094444.80: ** class ShadowController
40class ShadowController:
41 """A class to manage @shadow files"""
42 #@+others
43 #@+node:ekr.20080708094444.79: *3* x.ctor & x.reloadSettings
44 def __init__(self, c, trace=False, trace_writers=False):
45 """Ctor for ShadowController class."""
46 self.c = c
47 # Opcode dispatch dict.
48 self.dispatch_dict = {
49 'delete': self.op_delete,
50 'equal': self.op_equal,
51 'insert': self.op_insert,
52 'replace': self.op_replace,
53 }
54 # File encoding.
55 self.encoding = c.config.default_derived_file_encoding
56 # Configuration: set in reloadSettings.
57 self.shadow_subdir = None
58 self.shadow_prefix = None
59 self.shadow_in_home_dir = None
60 self.shadow_subdir = None
61 # Error handling...
62 self.errors = 0
63 self.last_error = '' # The last error message, regardless of whether it was actually shown.
64 self.trace = False
65 # Support for goto-line.
66 self.line_mapping = []
67 self.reloadSettings()
69 def reloadSettings(self):
70 """ShadowController.reloadSettings."""
71 c = self.c
72 self.shadow_subdir = c.config.getString('shadow-subdir') or '.leo_shadow'
73 self.shadow_prefix = c.config.getString('shadow-prefix') or ''
74 self.shadow_in_home_dir = c.config.getBool('shadow-in-home-dir', default=False)
75 self.shadow_subdir = g.os_path_normpath(self.shadow_subdir)
76 #@+node:ekr.20080711063656.1: *3* x.File utils
77 #@+node:ekr.20080711063656.7: *4* x.baseDirName
78 def baseDirName(self):
79 c = self.c
80 filename = c.fileName()
81 if filename:
82 return g.os_path_dirname(g.os_path_finalize(filename)) # 1341
83 print('')
84 self.error('Can not compute shadow path: .leo file has not been saved')
85 return None
86 #@+node:ekr.20080711063656.4: *4* x.dirName and pathName
87 def dirName(self, filename):
88 """Return the directory for filename."""
89 x = self
90 return g.os_path_dirname(x.pathName(filename))
92 def pathName(self, filename):
93 """Return the full path name of filename."""
94 x = self
95 theDir = x.baseDirName()
96 return theDir and g.os_path_finalize_join(theDir, filename) # 1341
97 #@+node:ekr.20080712080505.3: *4* x.isSignificantPublicFile
98 def isSignificantPublicFile(self, fn):
99 """
100 This tells the AtFile.read logic whether to import a public file
101 or use an existing public file.
102 """
103 return (
104 g.os_path_exists(fn) and
105 g.os_path_isfile(fn) and
106 g.os_path_getsize(fn) > 10
107 )
108 #@+node:ekr.20080710082231.19: *4* x.makeShadowDirectory
109 def makeShadowDirectory(self, fn):
110 """Make a shadow directory for the **public** fn."""
111 x = self
112 path = x.shadowDirName(fn)
113 if not g.os_path_exists(path):
114 # Force the creation of the directories.
115 g.makeAllNonExistentDirectories(path)
116 return g.os_path_exists(path) and g.os_path_isdir(path)
117 #@+node:ekr.20080713091247.1: *4* x.replaceFileWithString
118 def replaceFileWithString(self, encoding, fileName, s):
119 """
120 Replace the file with s if s is different from theFile's contents.
122 Return True if theFile was changed.
123 """
124 x, c = self, self.c
125 exists = g.os_path_exists(fileName)
126 if exists:
127 # Read the file. Return if it is the same.
128 s2, e = g.readFileIntoString(fileName)
129 if s2 is None:
130 return False
131 if s == s2:
132 report = c.config.getBool('report-unchanged-files', default=True)
133 if report and not g.unitTesting:
134 g.es('unchanged:', fileName)
135 return False
136 # Issue warning if directory does not exist.
137 theDir = g.os_path_dirname(fileName)
138 if theDir and not g.os_path_exists(theDir):
139 if not g.unitTesting:
140 x.error(f"not written: {fileName} directory not found")
141 return False
142 # Replace the file.
143 try:
144 with open(fileName, 'wb') as f:
145 # Fix bug 1243847: unicode error when saving @shadow nodes.
146 f.write(g.toEncodedString(s, encoding=encoding))
147 c.setFileTimeStamp(fileName) # Fix #1053. This is an *ancient* bug.
148 if not g.unitTesting:
149 kind = 'wrote' if exists else 'created'
150 g.es(f"{kind:>6}: {fileName}")
151 return True
152 except IOError:
153 x.error(f"unexpected exception writing file: {fileName}")
154 g.es_exception()
155 return False
156 #@+node:ekr.20080711063656.6: *4* x.shadowDirName and shadowPathName
157 def shadowDirName(self, filename):
158 """Return the directory for the shadow file corresponding to filename."""
159 x = self
160 return g.os_path_dirname(x.shadowPathName(filename))
162 def shadowPathName(self, filename):
163 """Return the full path name of filename, resolved using c.fileName()"""
164 x = self
165 c = x.c
166 baseDir = x.baseDirName()
167 fileDir = g.os_path_dirname(filename)
168 # 2011/01/26: bogomil: redirect shadow dir
169 if self.shadow_in_home_dir:
170 # Each .leo file has a separate shadow_cache in base dir
171 fname = "_".join(
172 [os.path.splitext(os.path.basename(c.mFileName))[0], "shadow_cache"])
173 # On Windows incorporate the drive letter to the private file path
174 if os.name == "nt":
175 fileDir = fileDir.replace(':', '%')
176 # build the chache path as a subdir of the base dir
177 fileDir = "/".join([baseDir, fname, fileDir])
178 return baseDir and g.os_path_finalize_join( # 1341
179 baseDir,
180 fileDir, # Bug fix: honor any directories specified in filename.
181 x.shadow_subdir,
182 x.shadow_prefix + g.shortFileName(filename))
183 #@+node:ekr.20080708192807.1: *3* x.Propagation
184 #@+node:ekr.20080708094444.35: *4* x.check_output
185 def check_output(self):
186 """Check that we produced a valid output."""
187 x = self
188 lines1 = x.b
189 junk, sents1 = x.separate_sentinels(x.old_sent_lines, x.marker)
190 lines2, sents2 = x.separate_sentinels(x.results, x.marker)
191 ok = lines1 == lines2 and sents1 == sents2
192 if g.unitTesting:
193 # The unit test will report the error.
194 return ok
195 if lines1 != lines2:
196 g.trace()
197 d = difflib.Differ()
198 aList = list(d.compare(lines2, x.b))
199 pprint.pprint(aList)
200 if sents1 != sents2:
201 x.show_error(
202 lines1=sents1,
203 lines2=sents2,
204 message="Sentinals not preserved!",
205 lines1_message="old sentinels",
206 lines2_message="new sentinels")
207 return ok
208 #@+node:ekr.20080708094444.38: *4* x.propagate_changed_lines (main algorithm) & helpers
209 def propagate_changed_lines(
210 self, new_public_lines, old_private_lines, marker, p=None):
211 #@+<< docstring >>
212 #@+node:ekr.20150207044400.9: *5* << docstring >>
213 """
214 The Mulder update algorithm, revised by EKR.
216 Use the diff between the old and new public lines to insperse sentinels
217 from old_private_lines into the result.
219 The algorithm never deletes or rearranges sentinels. However, verbatim
220 sentinels may be inserted or deleted as needed.
221 """
222 #@-<< docstring >>
223 x = self
224 x.init_ivars(new_public_lines, old_private_lines, marker)
225 sm = difflib.SequenceMatcher(None, x.a, x.b)
226 # Ensure leading sentinels are put first.
227 x.put_sentinels(0)
228 x.sentinels[0] = []
229 for tag, ai, aj, bi, bj in sm.get_opcodes():
230 f = x.dispatch_dict.get(tag, x.op_bad)
231 f(tag, ai, aj, bi, bj)
232 # Put the trailing sentinels & check the result.
233 x.results.extend(x.trailing_sentinels)
234 # check_output is likely to be more buggy than the code under test.
235 # x.check_output()
236 return x.results
237 #@+node:ekr.20150207111757.180: *5* x.dump_args
238 def dump_args(self):
239 """Dump the argument lines."""
240 x = self
241 table = (
242 (x.old_sent_lines, 'old private lines'),
243 (x.a, 'old public lines'),
244 (x.b, 'new public lines'),
245 )
246 for lines, title in table:
247 x.dump_lines(lines, title)
248 g.pr()
249 #@+node:ekr.20150207111757.178: *5* x.dump_lines
250 def dump_lines(self, lines, title):
251 """Dump the given lines."""
252 print(f"\n{title}...\n")
253 for i, line in enumerate(lines):
254 g.pr(f"{i:4} {line!r}")
255 #@+node:ekr.20150209044257.6: *5* x.init_data
256 def init_data(self):
257 """
258 Init x.sentinels and x.trailing_sentinels arrays.
259 Return the list of non-sentinel lines in x.old_sent_lines.
260 """
261 x = self
262 lines = x.old_sent_lines
263 # The sentinels preceding each non-sentinel line,
264 # not including @verbatim sentinels.
265 sentinels: List[str] = []
266 # A list of all non-sentinel lines found. Should match x.a.
267 new_lines = []
268 # A list of lists of sentinels preceding each line.
269 x.sentinels = []
270 i = 0
271 while i < len(lines):
272 line = lines[i]
273 i += 1
274 if x.marker.isVerbatimSentinel(line):
275 # Do *not* include the @verbatim sentinel.
276 if i < len(lines):
277 line = lines[i]
278 i += 1
279 x.sentinels.append(sentinels)
280 sentinels = []
281 new_lines.append(line)
282 else:
283 x.verbatim_error()
284 elif x.marker.isSentinel(line):
285 sentinels.append(line)
286 else:
287 x.sentinels.append(sentinels)
288 sentinels = []
289 new_lines.append(line)
290 x.trailing_sentinels = sentinels
291 return new_lines
292 #@+node:ekr.20080708094444.40: *5* x.init_ivars
293 def init_ivars(self, new_public_lines, old_private_lines, marker):
294 """Init all ivars used by propagate_changed_lines & its helpers."""
295 x = self
296 x.delim1, x.delim2 = marker.getDelims()
297 x.marker = marker
298 x.old_sent_lines = old_private_lines
299 x.results = []
300 x.verbatim_line = f"{x.delim1}@verbatim{x.delim2}\n"
301 old_public_lines = x.init_data()
302 x.b = x.preprocess(new_public_lines)
303 x.a = x.preprocess(old_public_lines)
304 #@+node:ekr.20150207044400.16: *5* x.op_bad
305 def op_bad(self, tag, ai, aj, bi, bj):
306 """Report an unexpected opcode."""
307 x = self
308 x.error(f"unknown SequenceMatcher opcode: {tag!r}")
309 #@+node:ekr.20150207044400.12: *5* x.op_delete
310 def op_delete(self, tag, ai, aj, bi, bj):
311 """Handle the 'delete' opcode."""
312 x = self
313 for i in range(ai, aj):
314 x.put_sentinels(i)
315 #@+node:ekr.20150207044400.13: *5* x.op_equal
316 def op_equal(self, tag, ai, aj, bi, bj):
317 """Handle the 'equal' opcode."""
318 x = self
319 assert aj - ai == bj - bi and x.a[ai:aj] == x.b[bi:bj]
320 for i in range(ai, aj):
321 x.put_sentinels(i)
322 # works because x.lines[ai:aj] == x.lines[bi:bj]
323 x.put_plain_line(x.a[i])
324 #@+node:ekr.20150207044400.14: *5* x.op_insert
325 def op_insert(self, tag, ai, aj, bi, bj):
326 """Handle the 'insert' opcode."""
327 x = self
328 for i in range(bi, bj):
329 x.put_plain_line(x.b[i])
330 # Prefer to put sentinels after inserted nodes.
331 # Requires a call to x.put_sentinels(0) before the main loop.
332 #@+node:ekr.20150207044400.15: *5* x.op_replace
333 def op_replace(self, tag, ai, aj, bi, bj):
334 """Handle the 'replace' opcode."""
335 x = self
336 if 1:
337 # Intersperse sentinels and lines.
338 b_lines = x.b[bi:bj]
339 for i in range(ai, aj):
340 x.put_sentinels(i)
341 if b_lines:
342 x.put_plain_line(b_lines.pop(0))
343 # Put any trailing lines.
344 while b_lines:
345 x.put_plain_line(b_lines.pop(0))
346 else:
347 # Feasible. Causes the present unit tests to fail.
348 for i in range(ai, aj):
349 x.put_sentinels(i)
350 for i in range(bi, bj):
351 x.put_plain_line(x.b[i])
352 #@+node:ekr.20150208060128.7: *5* x.preprocess
353 def preprocess(self, lines):
354 """
355 Preprocess public lines, adding newlines as needed.
356 This happens before the diff.
357 """
358 result = []
359 for line in lines:
360 if not line.endswith('\n'):
361 line = line + '\n'
362 result.append(line)
363 return result
364 #@+node:ekr.20150208223018.4: *5* x.put_plain_line
365 def put_plain_line(self, line):
366 """Put a plain line to x.results, inserting verbatim lines if necessary."""
367 x = self
368 if x.marker.isSentinel(line):
369 x.results.append(x.verbatim_line)
370 if x.trace:
371 print(f"put {repr(x.verbatim_line)}")
372 x.results.append(line)
373 if x.trace:
374 print(f"put {line!r}")
375 #@+node:ekr.20150209044257.8: *5* x.put_sentinels
376 def put_sentinels(self, i):
377 """Put all the sentinels to the results"""
378 x = self
379 if 0 <= i < len(x.sentinels):
380 sentinels = x.sentinels[i]
381 if x.trace:
382 g.trace(f"{i:3} {sentinels}")
383 x.results.extend(sentinels)
384 #@+node:ekr.20080708094444.36: *4* x.propagate_changes
385 def propagate_changes(self, old_public_file, old_private_file):
386 """
387 Propagate the changes from the public file (without_sentinels)
388 to the private file (with_sentinels)
389 """
390 x, at = self, self.c.atFileCommands
391 at.errors = 0
392 self.encoding = at.encoding
393 s = at.readFileToUnicode(old_private_file) # Sets at.encoding and inits at.readLines.
394 old_private_lines = g.splitLines(s or '') # #1466.
395 s = at.readFileToUnicode(old_public_file)
396 if at.encoding != self.encoding:
397 g.trace(f"can not happen: encoding mismatch: {at.encoding} {self.encoding}")
398 at.encoding = self.encoding
399 old_public_lines = g.splitLines(s)
400 if 0:
401 g.trace(f"\nprivate lines...{old_private_file}")
402 for s in old_private_lines:
403 g.trace(type(s), isinstance(s, str), repr(s))
404 g.trace(f"\npublic lines...{old_public_file}")
405 for s in old_public_lines:
406 g.trace(type(s), isinstance(s, str), repr(s))
407 marker = x.markerFromFileLines(old_private_lines, old_private_file)
408 new_private_lines = x.propagate_changed_lines(
409 old_public_lines, old_private_lines, marker)
410 # Never create the private file here!
411 fn = old_private_file
412 exists = g.os_path_exists(fn)
413 different = new_private_lines != old_private_lines
414 copy = exists and different
415 # 2010/01/07: check at.errors also.
416 if copy and x.errors == 0 and at.errors == 0:
417 s = ''.join(new_private_lines)
418 x.replaceFileWithString(at.encoding, fn, s)
419 return copy
420 #@+node:bwmulder.20041231170726: *4* x.updatePublicAndPrivateFiles
421 def updatePublicAndPrivateFiles(self, root, fn, shadow_fn):
422 """handle crucial @shadow read logic.
424 This will be called only if the public and private files both exist."""
425 x = self
426 if x.isSignificantPublicFile(fn):
427 # Update the private shadow file from the public file.
428 written = x.propagate_changes(fn, shadow_fn)
429 if written:
430 x.message(f"updated private {shadow_fn} from public {fn}")
431 else:
432 pass
433 # Don't write *anything*.
434 # if 0: # This causes considerable problems.
435 # # Create the public file from the private shadow file.
436 # x.copy_file_removing_sentinels(shadow_fn,fn)
437 # x.message("created public %s from private %s " % (fn, shadow_fn))
438 #@+node:ekr.20080708094444.89: *3* x.Utils...
439 #@+node:ekr.20080708094444.85: *4* x.error & message & verbatim_error
440 def error(self, s, silent=False):
441 x = self
442 if not silent:
443 g.error(s)
444 # For unit testing.
445 x.last_error = s
446 x.errors += 1
448 def message(self, s):
449 g.es_print(s, color='orange')
451 def verbatim_error(self):
452 x = self
453 x.error('file syntax error: nothing follows verbatim sentinel')
454 g.trace(g.callers())
455 #@+node:ekr.20090529125512.6122: *4* x.markerFromFileLines & helper
456 def markerFromFileLines(self, lines, fn):
457 """Return the sentinel delimiter comment to be used for filename."""
458 at, x = self.c.atFileCommands, self
459 s = x.findLeoLine(lines)
460 ok, junk, start, end, junk = at.parseLeoSentinel(s)
461 if end:
462 delims = '', start, end
463 else:
464 delims = start, '', ''
465 return x.Marker(delims)
466 #@+node:ekr.20090529125512.6125: *5* x.findLeoLine
467 def findLeoLine(self, lines):
468 """Return the @+leo line, or ''."""
469 for line in lines:
470 i = line.find('@+leo')
471 if i != -1:
472 return line
473 return ''
474 #@+node:ekr.20080708094444.9: *4* x.markerFromFileName
475 def markerFromFileName(self, filename):
476 """Return the sentinel delimiter comment to be used for filename."""
477 x = self
478 if not filename:
479 return None
480 root, ext = g.os_path_splitext(filename)
481 if ext == '.tmp':
482 root, ext = os.path.splitext(root)
483 delims = g.comment_delims_from_extension(filename)
484 marker = x.Marker(delims)
485 return marker
486 #@+node:ekr.20080708094444.29: *4* x.separate_sentinels
487 def separate_sentinels(self, lines, marker):
488 """
489 Separates regular lines from sentinel lines.
490 Do not return @verbatim sentinels.
492 Returns (regular_lines, sentinel_lines)
493 """
494 x = self
495 regular_lines = []
496 sentinel_lines = []
497 i = 0
498 while i < len(lines):
499 line = lines[i]
500 if marker.isVerbatimSentinel(line):
501 # Add the plain line that *looks* like a sentinel,
502 # but *not* the preceding @verbatim sentinel itself.
503 # Adding the actual sentinel would spoil the sentinel test when
504 # the user adds or deletes a line requiring an @verbatim line.
505 i += 1
506 if i < len(lines):
507 line = lines[i]
508 regular_lines.append(line)
509 else:
510 x.verbatim_error()
511 elif marker.isSentinel(line):
512 sentinel_lines.append(line)
513 else:
514 regular_lines.append(line)
515 i += 1
516 return regular_lines, sentinel_lines
517 #@+node:ekr.20080708094444.33: *4* x.show_error & helper
518 def show_error(self, lines1, lines2, message, lines1_message, lines2_message):
519 x = self
520 banner1 = '=' * 30
521 banner2 = '-' * 30
522 g.es_print(f"{banner1}\n{message}\n{banner1}\n{lines1_message}\n{banner2}")
523 x.show_error_lines(lines1, 'shadow_errors.tmp1')
524 g.es_print(f"\n{banner1}\n{lines2_message}\n{banner1}")
525 x.show_error_lines(lines2, 'shadow_errors.tmp2')
526 g.es_print('\n@shadow did not pick up the external changes correctly')
527 # g.es_print('Please check shadow.tmp1 and shadow.tmp2 for differences')
528 #@+node:ekr.20080822065427.4: *5* x.show_error_lines
529 def show_error_lines(self, lines, fileName):
530 for i, line in enumerate(lines):
531 g.es_print(f"{i:3} {line!r}")
532 if False: # Only for major debugging.
533 try:
534 f1 = open(fileName, "w")
535 for s in lines:
536 f1.write(repr(s))
537 f1.close()
538 except IOError:
539 g.es_exception()
540 g.es_print('can not open', fileName)
541 #@+node:ekr.20090529061522.5727: *3* class x.Marker
542 class Marker:
543 """A class representing comment delims in @shadow files."""
544 #@+others
545 #@+node:ekr.20090529061522.6257: *4* ctor & repr
546 def __init__(self, delims):
547 """Ctor for Marker class."""
548 delim1, delim2, delim3 = delims
549 self.delim1 = delim1 # Single-line comment delim.
550 self.delim2 = delim2 # Block comment starting delim.
551 self.delim3 = delim3 # Block comment ending delim.
552 if not delim1 and not delim2:
553 # if g.unitTesting:
554 # assert False,repr(delims)
555 self.delim1 = g.app.language_delims_dict.get('unknown_language')
557 def __repr__(self):
558 if self.delim1:
559 delims = self.delim1
560 else:
561 delims = f"{self.delim2} {self.delim3}"
562 return f"<Marker: delims: {delims!r}>"
563 #@+node:ekr.20090529061522.6258: *4* getDelims
564 def getDelims(self):
565 """Return the pair of delims to be used in sentinel lines."""
566 if self.delim1:
567 return self.delim1, ''
568 return self.delim2, self.delim3
569 #@+node:ekr.20090529061522.6259: *4* isSentinel
570 def isSentinel(self, s, suffix=''):
571 """Return True is line s contains a valid sentinel comment."""
572 s = s.strip()
573 if self.delim1 and s.startswith(self.delim1):
574 return s.startswith(self.delim1 + '@' + suffix)
575 if self.delim2:
576 return s.startswith(
577 self.delim2 + '@' + suffix) and s.endswith(self.delim3)
578 return False
579 #@+node:ekr.20090529061522.6260: *4* isVerbatimSentinel
580 def isVerbatimSentinel(self, s):
581 """Return True if s is an @verbatim sentinel."""
582 return self.isSentinel(s, suffix='verbatim')
583 #@-others
584 #@-others
585#@-others
586#@@language python
587#@@tabwidth -4
588#@@pagewidth 70
589#@-leo