1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Spans staged data among multiple discs
40
41 This is the Cedar Backup span tool. It is intended for use by people who stage
42 more data than can fit on a single disc. It allows a user to split staged data
43 among more than one disc. It can't be an extension because it requires user
44 input when switching media.
45
46 Most configuration is taken from the Cedar Backup configuration file,
47 specifically the store section. A few pieces of configuration are taken
48 directly from the user.
49
50 @author: Kenneth J. Pronovici <pronovic@ieee.org>
51 """
52
53
54
55
56
57
58 import sys
59 import os
60 import logging
61 import tempfile
62
63
64 from CedarBackup2.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT
65 from CedarBackup2.util import displayBytes, convertSize, mount, unmount
66 from CedarBackup2.util import UNIT_SECTORS, UNIT_BYTES
67 from CedarBackup2.config import Config
68 from CedarBackup2.filesystem import BackupFileList, compareDigestMaps, normalizeDir
69 from CedarBackup2.cli import Options, setupLogging, setupPathResolver
70 from CedarBackup2.cli import DEFAULT_CONFIG, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE
71 from CedarBackup2.actions.constants import STORE_INDICATOR
72 from CedarBackup2.actions.util import createWriter
73 from CedarBackup2.actions.store import writeIndicatorFile
74 from CedarBackup2.actions.util import findDailyDirs
75 from CedarBackup2.util import Diagnostics
76
77
78
79
80
81
82 logger = logging.getLogger("CedarBackup2.log.tools.span")
83
84
85
86
87
88
90
91 """
92 Tool-specific command-line options.
93
94 Most of the cback command-line options are exactly what we need here --
95 logfile path, permissions, verbosity, etc. However, we need to make a few
96 tweaks since we don't accept any actions.
97
98 Also, a few extra command line options that we accept are really ignored
99 underneath. I just don't care about that for a tool like this.
100 """
101
103 """
104 Validates command-line options represented by the object.
105 There are no validations here, because we don't use any actions.
106 @raise ValueError: If one of the validations fails.
107 """
108 pass
109
110
111
112
113
114
115
116
117
118
120 """
121 Implements the command-line interface for the C{cback-span} script.
122
123 Essentially, this is the "main routine" for the cback-span script. It does
124 all of the argument processing for the script, and then also implements the
125 tool functionality.
126
127 This function looks pretty similiar to C{CedarBackup2.cli.cli()}. It's not
128 easy to refactor this code to make it reusable and also readable, so I've
129 decided to just live with the duplication.
130
131 A different error code is returned for each type of failure:
132
133 - C{1}: The Python interpreter version is < 2.7
134 - C{2}: Error processing command-line arguments
135 - C{3}: Error configuring logging
136 - C{4}: Error parsing indicated configuration file
137 - C{5}: Backup was interrupted with a CTRL-C or similar
138 - C{6}: Error executing other parts of the script
139
140 @note: This script uses print rather than logging to the INFO level, because
141 it is interactive. Underlying Cedar Backup functionality uses the logging
142 mechanism exclusively.
143
144 @return: Error code as described above.
145 """
146 try:
147 if map(int, [sys.version_info[0], sys.version_info[1]]) < [2, 7]:
148 sys.stderr.write("Python 2 version 2.7 or greater required.\n")
149 return 1
150 except:
151
152 sys.stderr.write("Python 2 version 2.7 or greater required.\n")
153 return 1
154
155 try:
156 options = SpanOptions(argumentList=sys.argv[1:])
157 except Exception, e:
158 _usage()
159 sys.stderr.write(" *** Error: %s\n" % e)
160 return 2
161
162 if options.help:
163 _usage()
164 return 0
165 if options.version:
166 _version()
167 return 0
168 if options.diagnostics:
169 _diagnostics()
170 return 0
171
172 if options.stacktrace:
173 logfile = setupLogging(options)
174 else:
175 try:
176 logfile = setupLogging(options)
177 except Exception as e:
178 sys.stderr.write("Error setting up logging: %s\n" % e)
179 return 3
180
181 logger.info("Cedar Backup 'span' utility run started.")
182 logger.info("Options were [%s]", options)
183 logger.info("Logfile is [%s]", logfile)
184
185 if options.config is None:
186 logger.debug("Using default configuration file.")
187 configPath = DEFAULT_CONFIG
188 else:
189 logger.debug("Using user-supplied configuration file.")
190 configPath = options.config
191
192 try:
193 logger.info("Configuration path is [%s]", configPath)
194 config = Config(xmlPath=configPath)
195 setupPathResolver(config)
196 except Exception, e:
197 logger.error("Error reading or handling configuration: %s", e)
198 logger.info("Cedar Backup 'span' utility run completed with status 4.")
199 return 4
200
201 if options.stacktrace:
202 _executeAction(options, config)
203 else:
204 try:
205 _executeAction(options, config)
206 except KeyboardInterrupt:
207 logger.error("Backup interrupted.")
208 logger.info("Cedar Backup 'span' utility run completed with status 5.")
209 return 5
210 except Exception, e:
211 logger.error("Error executing backup: %s", e)
212 logger.info("Cedar Backup 'span' utility run completed with status 6.")
213 return 6
214
215 logger.info("Cedar Backup 'span' utility run completed with status 0.")
216 return 0
217
218
219
220
221
222
223
224
225
226
228 """
229 Prints usage information for the cback script.
230 @param fd: File descriptor used to print information.
231 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
232 """
233 fd.write("\n")
234 fd.write(" Usage: cback-span [switches]\n")
235 fd.write("\n")
236 fd.write(" Cedar Backup 'span' tool.\n")
237 fd.write("\n")
238 fd.write(" This Cedar Backup utility spans staged data between multiple discs.\n")
239 fd.write(" It is a utility, not an extension, and requires user interaction.\n")
240 fd.write("\n")
241 fd.write(" The following switches are accepted, mostly to set up underlying\n")
242 fd.write(" Cedar Backup functionality:\n")
243 fd.write("\n")
244 fd.write(" -h, --help Display this usage/help listing\n")
245 fd.write(" -V, --version Display version information\n")
246 fd.write(" -b, --verbose Print verbose output as well as logging to disk\n")
247 fd.write(" -c, --config Path to config file (default: %s)\n" % DEFAULT_CONFIG)
248 fd.write(" -l, --logfile Path to logfile (default: %s)\n" % DEFAULT_LOGFILE)
249 fd.write(" -o, --owner Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1]))
250 fd.write(" -m, --mode Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE)
251 fd.write(" -O, --output Record some sub-command (i.e. tar) output to the log\n")
252 fd.write(" -d, --debug Write debugging information to the log (implies --output)\n")
253 fd.write(" -s, --stack Dump a Python stack trace instead of swallowing exceptions\n")
254 fd.write("\n")
255
256
257
258
259
260
262 """
263 Prints version information for the cback script.
264 @param fd: File descriptor used to print information.
265 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
266 """
267 fd.write("\n")
268 fd.write(" Cedar Backup 'span' tool.\n")
269 fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE))
270 fd.write("\n")
271 fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL))
272 fd.write(" See CREDITS for a list of included code and other contributors.\n")
273 fd.write(" This is free software; there is NO warranty. See the\n")
274 fd.write(" GNU General Public License version 2 for copying conditions.\n")
275 fd.write("\n")
276 fd.write(" Use the --help option for usage information.\n")
277 fd.write("\n")
278
279
280
281
282
283
285 """
286 Prints runtime diagnostics information.
287 @param fd: File descriptor used to print information.
288 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
289 """
290 fd.write("\n")
291 fd.write("Diagnostics:\n")
292 fd.write("\n")
293 Diagnostics().printDiagnostics(fd=fd, prefix=" ")
294 fd.write("\n")
295
296
297
298
299
300
302 """
303 Implements the guts of the cback-span tool.
304
305 @param options: Program command-line options.
306 @type options: SpanOptions object.
307
308 @param config: Program configuration.
309 @type config: Config object.
310
311 @raise Exception: Under many generic error conditions
312 """
313 print ""
314 print "================================================"
315 print " Cedar Backup 'span' tool"
316 print "================================================"
317 print ""
318 print "This the Cedar Backup span tool. It is used to split up staging"
319 print "data when that staging data does not fit onto a single disc."
320 print ""
321 print "This utility operates using Cedar Backup configuration. Configuration"
322 print "specifies which staging directory to look at and which writer device"
323 print "and media type to use."
324 print ""
325 if not _getYesNoAnswer("Continue?", default="Y"):
326 return
327 print "==="
328
329 print ""
330 print "Cedar Backup store configuration looks like this:"
331 print ""
332 print " Source Directory...: %s" % config.store.sourceDir
333 print " Media Type.........: %s" % config.store.mediaType
334 print " Device Type........: %s" % config.store.deviceType
335 print " Device Path........: %s" % config.store.devicePath
336 print " Device SCSI ID.....: %s" % config.store.deviceScsiId
337 print " Drive Speed........: %s" % config.store.driveSpeed
338 print " Check Data Flag....: %s" % config.store.checkData
339 print " No Eject Flag......: %s" % config.store.noEject
340 print ""
341 if not _getYesNoAnswer("Is this OK?", default="Y"):
342 return
343 print "==="
344
345 (writer, mediaCapacity) = _getWriter(config)
346
347 print ""
348 print "Please wait, indexing the source directory (this may take a while)..."
349 (dailyDirs, fileList) = _findDailyDirs(config.store.sourceDir)
350 print "==="
351
352 print ""
353 print "The following daily staging directories have not yet been written to disc:"
354 print ""
355 for dailyDir in dailyDirs:
356 print " %s" % dailyDir
357
358 totalSize = fileList.totalSize()
359 print ""
360 print "The total size of the data in these directories is %s." % displayBytes(totalSize)
361 print ""
362 if not _getYesNoAnswer("Continue?", default="Y"):
363 return
364 print "==="
365
366 print ""
367 print "Based on configuration, the capacity of your media is %s." % displayBytes(mediaCapacity)
368
369 print ""
370 print "Since estimates are not perfect and there is some uncertainly in"
371 print "media capacity calculations, it is good to have a \"cushion\","
372 print "a percentage of capacity to set aside. The cushion reduces the"
373 print "capacity of your media, so a 1.5% cushion leaves 98.5% remaining."
374 print ""
375 cushion = _getFloat("What cushion percentage?", default=4.5)
376 print "==="
377
378 realCapacity = ((100.0 - cushion)/100.0) * mediaCapacity
379 minimumDiscs = (totalSize/realCapacity) + 1
380 print ""
381 print "The real capacity, taking into account the %.2f%% cushion, is %s." % (cushion, displayBytes(realCapacity))
382 print "It will take at least %d disc(s) to store your %s of data." % (minimumDiscs, displayBytes(totalSize))
383 print ""
384 if not _getYesNoAnswer("Continue?", default="Y"):
385 return
386 print "==="
387
388 happy = False
389 while not happy:
390 print ""
391 print "Which algorithm do you want to use to span your data across"
392 print "multiple discs?"
393 print ""
394 print "The following algorithms are available:"
395 print ""
396 print " first....: The \"first-fit\" algorithm"
397 print " best.....: The \"best-fit\" algorithm"
398 print " worst....: The \"worst-fit\" algorithm"
399 print " alternate: The \"alternate-fit\" algorithm"
400 print ""
401 print "If you don't like the results you will have a chance to try a"
402 print "different one later."
403 print ""
404 algorithm = _getChoiceAnswer("Which algorithm?", "worst", [ "first", "best", "worst", "alternate", ])
405 print "==="
406
407 print ""
408 print "Please wait, generating file lists (this may take a while)..."
409 spanSet = fileList.generateSpan(capacity=realCapacity, algorithm="%s_fit" % algorithm)
410 print "==="
411
412 print ""
413 print "Using the \"%s-fit\" algorithm, Cedar Backup can split your data" % algorithm
414 print "into %d discs." % len(spanSet)
415 print ""
416 counter = 0
417 for item in spanSet:
418 counter += 1
419 print "Disc %d: %d files, %s, %.2f%% utilization" % (counter, len(item.fileList),
420 displayBytes(item.size), item.utilization)
421 print ""
422 if _getYesNoAnswer("Accept this solution?", default="Y"):
423 happy = True
424 print "==="
425
426 counter = 0
427 for spanItem in spanSet:
428 counter += 1
429 if counter == 1:
430 print ""
431 _getReturn("Please place the first disc in your backup device.\nPress return when ready.")
432 print "==="
433 else:
434 print ""
435 _getReturn("Please replace the disc in your backup device.\nPress return when ready.")
436 print "==="
437 _writeDisc(config, writer, spanItem)
438
439 _writeStoreIndicator(config, dailyDirs)
440
441 print ""
442 print "Completed writing all discs."
443
444
445
446
447
448
450 """
451 Returns a list of all daily staging directories that have not yet been
452 stored.
453
454 The store indicator file C{cback.store} will be written to a daily staging
455 directory once that directory is written to disc. So, this function looks
456 at each daily staging directory within the configured staging directory, and
457 returns a list of those which do not contain the indicator file.
458
459 Returned is a tuple containing two items: a list of daily staging
460 directories, and a BackupFileList containing all files among those staging
461 directories.
462
463 @param stagingDir: Configured staging directory
464
465 @return: Tuple (staging dirs, backup file list)
466 """
467 results = findDailyDirs(stagingDir, STORE_INDICATOR)
468 fileList = BackupFileList()
469 for item in results:
470 fileList.addDirContents(item)
471 return (results, fileList)
472
473
474
475
476
477
489
490
491
492
493
494
505
506
507
508
509
510
524
526 """
527 Initialize an ISO image for a span item.
528 @param config: Cedar Backup configuration
529 @param writer: Writer to use
530 @param spanItem: Span item to write
531 """
532 complete = False
533 while not complete:
534 try:
535 print "Initializing image..."
536 writer.initializeImage(newDisc=True, tmpdir=config.options.workingDir)
537 for path in spanItem.fileList:
538 graftPoint = os.path.dirname(path.replace(config.store.sourceDir, "", 1))
539 writer.addImageEntry(path, graftPoint)
540 complete = True
541 except KeyboardInterrupt, e:
542 raise e
543 except Exception, e:
544 logger.error("Failed to initialize image: %s", e)
545 if not _getYesNoAnswer("Retry initialization step?", default="Y"):
546 raise e
547 print "Ok, attempting retry."
548 print "==="
549 print "Completed initializing image."
550
552 """
553 Writes a ISO image for a span item.
554 @param config: Cedar Backup configuration
555 @param writer: Writer to use
556 """
557 complete = False
558 while not complete:
559 try:
560 print "Writing image to disc..."
561 writer.writeImage()
562 complete = True
563 except KeyboardInterrupt, e:
564 raise e
565 except Exception, e:
566 logger.error("Failed to write image: %s", e)
567 if not _getYesNoAnswer("Retry this step?", default="Y"):
568 raise e
569 print "Ok, attempting retry."
570 _getReturn("Please replace media if needed.\nPress return when ready.")
571 print "==="
572 print "Completed writing image."
573
575 """
576 Run a consistency check on an ISO image for a span item.
577 @param config: Cedar Backup configuration
578 @param writer: Writer to use
579 @param spanItem: Span item to write
580 """
581 if config.store.checkData:
582 complete = False
583 while not complete:
584 try:
585 print "Running consistency check..."
586 _consistencyCheck(config, spanItem.fileList)
587 complete = True
588 except KeyboardInterrupt, e:
589 raise e
590 except Exception, e:
591 logger.error("Consistency check failed: %s", e)
592 if not _getYesNoAnswer("Retry the consistency check?", default="Y"):
593 raise e
594 if _getYesNoAnswer("Rewrite the disc first?", default="N"):
595 print "Ok, attempting retry."
596 _getReturn("Please replace the disc in your backup device.\nPress return when ready.")
597 print "==="
598 _discWriteImage(config, writer)
599 else:
600 print "Ok, attempting retry."
601 print "==="
602 print "Completed consistency check."
603
604
605
606
607
608
610 """
611 Runs a consistency check against media in the backup device.
612
613 The function mounts the device at a temporary mount point in the working
614 directory, and then compares the passed-in file list's digest map with the
615 one generated from the disc. The two lists should be identical.
616
617 If no exceptions are thrown, there were no problems with the consistency
618 check.
619
620 @warning: The implementation of this function is very UNIX-specific.
621
622 @param config: Config object.
623 @param fileList: BackupFileList whose contents to check against
624
625 @raise ValueError: If the check fails
626 @raise IOError: If there is a problem working with the media.
627 """
628 logger.debug("Running consistency check.")
629 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir)
630 try:
631 mount(config.store.devicePath, mountPoint, "iso9660")
632 discList = BackupFileList()
633 discList.addDirContents(mountPoint)
634 sourceList = BackupFileList()
635 sourceList.extend(fileList)
636 discListDigest = discList.generateDigestMap(stripPrefix=normalizeDir(mountPoint))
637 sourceListDigest = sourceList.generateDigestMap(stripPrefix=normalizeDir(config.store.sourceDir))
638 compareDigestMaps(sourceListDigest, discListDigest, verbose=True)
639 logger.info("Consistency check completed. No problems found.")
640 finally:
641 unmount(mountPoint, True, 5, 1)
642
643
644
645
646
647
649 """
650 Get a yes/no answer from the user.
651 The default will be placed at the end of the prompt.
652 A "Y" or "y" is considered yes, anything else no.
653 A blank (empty) response results in the default.
654 @param prompt: Prompt to show.
655 @param default: Default to set if the result is blank
656 @return: Boolean true/false corresponding to Y/N
657 """
658 if default == "Y":
659 prompt = "%s [Y/n]: " % prompt
660 else:
661 prompt = "%s [y/N]: " % prompt
662 answer = raw_input(prompt)
663 if answer in [ None, "", ]:
664 answer = default
665 if answer[0] in [ "Y", "y", ]:
666 return True
667 else:
668 return False
669
671 """
672 Get a particular choice from the user.
673 The default will be placed at the end of the prompt.
674 The function loops until getting a valid choice.
675 A blank (empty) response results in the default.
676 @param prompt: Prompt to show.
677 @param default: Default to set if the result is None or blank.
678 @param validChoices: List of valid choices (strings)
679 @return: Valid choice from user.
680 """
681 prompt = "%s [%s]: " % (prompt, default)
682 answer = raw_input(prompt)
683 if answer in [ None, "", ]:
684 answer = default
685 while answer not in validChoices:
686 print "Choice must be one of %s" % validChoices
687 answer = raw_input(prompt)
688 return answer
689
691 """
692 Get a floating point number from the user.
693 The default will be placed at the end of the prompt.
694 The function loops until getting a valid floating point number.
695 A blank (empty) response results in the default.
696 @param prompt: Prompt to show.
697 @param default: Default to set if the result is None or blank.
698 @return: Floating point number from user
699 """
700 prompt = "%s [%.2f]: " % (prompt, default)
701 while True:
702 answer = raw_input(prompt)
703 if answer in [ None, "" ]:
704 return default
705 else:
706 try:
707 return float(answer)
708 except ValueError:
709 print "Enter a floating point number."
710
712 """
713 Get a return key from the user.
714 @param prompt: Prompt to show.
715 """
716 raw_input(prompt)
717
718
719
720
721
722
723 if __name__ == "__main__":
724 sys.exit(cli())
725