Package CedarBackup2 :: Package extend :: Module postgresql
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup2.extend.postgresql

  1  # -*- coding: iso-8859-1 -*- 
  2  # vim: set ft=python ts=3 sw=3 expandtab: 
  3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  4  # 
  5  #              C E D A R 
  6  #          S O L U T I O N S       "Software done right." 
  7  #           S O F T W A R E 
  8  # 
  9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 10  # 
 11  # Copyright (c) 2006,2010 Kenneth J. Pronovici. 
 12  # Copyright (c) 2006 Antoine Beaupre. 
 13  # All rights reserved. 
 14  # 
 15  # This program is free software; you can redistribute it and/or 
 16  # modify it under the terms of the GNU General Public License, 
 17  # Version 2, as published by the Free Software Foundation. 
 18  # 
 19  # This program is distributed in the hope that it will be useful, 
 20  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 21  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 22  # 
 23  # Copies of the GNU General Public License are available from 
 24  # the Free Software Foundation website, http://www.gnu.org/. 
 25  # 
 26  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 27  # 
 28  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
 29  #            Antoine Beaupre <anarcat@koumbit.org> 
 30  # Language : Python 2 (>= 2.7) 
 31  # Project  : Official Cedar Backup Extensions 
 32  # Purpose  : Provides an extension to back up PostgreSQL databases. 
 33  # 
 34  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 35  # This file was created with a width of 132 characters, and NO tabs. 
 36   
 37  ######################################################################## 
 38  # Module documentation 
 39  ######################################################################## 
 40   
 41  """ 
 42  Provides an extension to back up PostgreSQL databases. 
 43   
 44  This is a Cedar Backup extension used to back up PostgreSQL databases via the 
 45  Cedar Backup command line.  It requires a new configurations section 
 46  <postgresql> and is intended to be run either immediately before or immediately 
 47  after the standard collect action.  Aside from its own configuration, it 
 48  requires the options and collect configuration sections in the standard Cedar 
 49  Backup configuration file. 
 50   
 51  The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with 
 52  the PostgreSQL product.  Output can be compressed using C{gzip} or C{bzip2}. 
 53  Administrators can configure the extension either to back up all databases or 
 54  to back up only specific databases.  The extension assumes that the current 
 55  user has passwordless access to the database since there is no easy way to pass 
 56  a password to the C{pg_dump} client. This can be accomplished using appropriate 
 57  voodoo in the C{pg_hda.conf} file. 
 58   
 59  Note that this code always produces a full backup.  There is currently no 
 60  facility for making incremental backups. 
 61   
 62  You should always make C{/etc/cback.conf} unreadble to non-root users once you 
 63  place postgresql configuration into it, since postgresql configuration will 
 64  contain information about available PostgreSQL databases and usernames. 
 65   
 66  Use of this extension I{may} expose usernames in the process listing (via 
 67  C{ps}) when the backup is running if the username is specified in the 
 68  configuration. 
 69   
 70  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 71  @author: Antoine Beaupre <anarcat@koumbit.org> 
 72  """ 
 73   
 74  ######################################################################## 
 75  # Imported modules 
 76  ######################################################################## 
 77   
 78  # System modules 
 79  import os 
 80  import logging 
 81  from gzip import GzipFile 
 82  from bz2 import BZ2File 
 83   
 84  # Cedar Backup modules 
 85  from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode 
 86  from CedarBackup2.xmlutil import readFirstChild, readString, readStringList, readBoolean 
 87  from CedarBackup2.config import VALID_COMPRESS_MODES 
 88  from CedarBackup2.util import resolveCommand, executeCommand 
 89  from CedarBackup2.util import ObjectTypeList, changeOwnership 
 90   
 91   
 92  ######################################################################## 
 93  # Module-wide constants and variables 
 94  ######################################################################## 
 95   
 96  logger = logging.getLogger("CedarBackup2.log.extend.postgresql") 
 97  POSTGRESQLDUMP_COMMAND = [ "pg_dump", ] 
 98  POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ] 
99 100 101 ######################################################################## 102 # PostgresqlConfig class definition 103 ######################################################################## 104 105 -class PostgresqlConfig(object):
106 107 """ 108 Class representing PostgreSQL configuration. 109 110 The PostgreSQL configuration information is used for backing up PostgreSQL databases. 111 112 The following restrictions exist on data in this class: 113 114 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}. 115 - The 'all' flag must be 'Y' if no databases are defined. 116 - The 'all' flag must be 'N' if any databases are defined. 117 - Any values in the databases list must be strings. 118 119 @sort: __init__, __repr__, __str__, __cmp__, user, all, databases 120 """ 121
122 - def __init__(self, user=None, compressMode=None, all=None, databases=None): # pylint: disable=W0622
123 """ 124 Constructor for the C{PostgresqlConfig} class. 125 126 @param user: User to execute backup as. 127 @param compressMode: Compress mode for backed-up files. 128 @param all: Indicates whether to back up all databases. 129 @param databases: List of databases to back up. 130 """ 131 self._user = None 132 self._compressMode = None 133 self._all = None 134 self._databases = None 135 self.user = user 136 self.compressMode = compressMode 137 self.all = all 138 self.databases = databases
139
140 - def __repr__(self):
141 """ 142 Official string representation for class instance. 143 """ 144 return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
145
146 - def __str__(self):
147 """ 148 Informal string representation for class instance. 149 """ 150 return self.__repr__()
151
152 - def __cmp__(self, other):
153 """ 154 Definition of equals operator for this class. 155 @param other: Other object to compare to. 156 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 157 """ 158 if other is None: 159 return 1 160 if self.user != other.user: 161 if self.user < other.user: 162 return -1 163 else: 164 return 1 165 if self.compressMode != other.compressMode: 166 if self.compressMode < other.compressMode: 167 return -1 168 else: 169 return 1 170 if self.all != other.all: 171 if self.all < other.all: 172 return -1 173 else: 174 return 1 175 if self.databases != other.databases: 176 if self.databases < other.databases: 177 return -1 178 else: 179 return 1 180 return 0
181
182 - def _setUser(self, value):
183 """ 184 Property target used to set the user value. 185 """ 186 if value is not None: 187 if len(value) < 1: 188 raise ValueError("User must be non-empty string.") 189 self._user = value
190
191 - def _getUser(self):
192 """ 193 Property target used to get the user value. 194 """ 195 return self._user
196
197 - def _setCompressMode(self, value):
198 """ 199 Property target used to set the compress mode. 200 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}. 201 @raise ValueError: If the value is not valid. 202 """ 203 if value is not None: 204 if value not in VALID_COMPRESS_MODES: 205 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES) 206 self._compressMode = value
207
208 - def _getCompressMode(self):
209 """ 210 Property target used to get the compress mode. 211 """ 212 return self._compressMode
213
214 - def _setAll(self, value):
215 """ 216 Property target used to set the 'all' flag. 217 No validations, but we normalize the value to C{True} or C{False}. 218 """ 219 if value: 220 self._all = True 221 else: 222 self._all = False
223
224 - def _getAll(self):
225 """ 226 Property target used to get the 'all' flag. 227 """ 228 return self._all
229
230 - def _setDatabases(self, value):
231 """ 232 Property target used to set the databases list. 233 Either the value must be C{None} or each element must be a string. 234 @raise ValueError: If the value is not a string. 235 """ 236 if value is None: 237 self._databases = None 238 else: 239 for database in value: 240 if len(database) < 1: 241 raise ValueError("Each database must be a non-empty string.") 242 try: 243 saved = self._databases 244 self._databases = ObjectTypeList(basestring, "string") 245 self._databases.extend(value) 246 except Exception, e: 247 self._databases = saved 248 raise e
249
250 - def _getDatabases(self):
251 """ 252 Property target used to get the databases list. 253 """ 254 return self._databases
255 256 user = property(_getUser, _setUser, None, "User to execute backup as.") 257 compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.") 258 all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.") 259 databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.") 260
261 262 ######################################################################## 263 # LocalConfig class definition 264 ######################################################################## 265 266 -class LocalConfig(object):
267 268 """ 269 Class representing this extension's configuration document. 270 271 This is not a general-purpose configuration object like the main Cedar 272 Backup configuration object. Instead, it just knows how to parse and emit 273 PostgreSQL-specific configuration values. Third parties who need to read and 274 write configuration related to this extension should access it through the 275 constructor, C{validate} and C{addConfig} methods. 276 277 @note: Lists within this class are "unordered" for equality comparisons. 278 279 @sort: __init__, __repr__, __str__, __cmp__, postgresql, validate, addConfig 280 """ 281
282 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
283 """ 284 Initializes a configuration object. 285 286 If you initialize the object without passing either C{xmlData} or 287 C{xmlPath} then configuration will be empty and will be invalid until it 288 is filled in properly. 289 290 No reference to the original XML data or original path is saved off by 291 this class. Once the data has been parsed (successfully or not) this 292 original information is discarded. 293 294 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate} 295 method will be called (with its default arguments) against configuration 296 after successfully parsing any passed-in XML. Keep in mind that even if 297 C{validate} is C{False}, it might not be possible to parse the passed-in 298 XML document if lower-level validations fail. 299 300 @note: It is strongly suggested that the C{validate} option always be set 301 to C{True} (the default) unless there is a specific need to read in 302 invalid configuration from disk. 303 304 @param xmlData: XML data representing configuration. 305 @type xmlData: String data. 306 307 @param xmlPath: Path to an XML file on disk. 308 @type xmlPath: Absolute path to a file on disk. 309 310 @param validate: Validate the document after parsing it. 311 @type validate: Boolean true/false. 312 313 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in. 314 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed. 315 @raise ValueError: If the parsed configuration document is not valid. 316 """ 317 self._postgresql = None 318 self.postgresql = None 319 if xmlData is not None and xmlPath is not None: 320 raise ValueError("Use either xmlData or xmlPath, but not both.") 321 if xmlData is not None: 322 self._parseXmlData(xmlData) 323 if validate: 324 self.validate() 325 elif xmlPath is not None: 326 xmlData = open(xmlPath).read() 327 self._parseXmlData(xmlData) 328 if validate: 329 self.validate()
330
331 - def __repr__(self):
332 """ 333 Official string representation for class instance. 334 """ 335 return "LocalConfig(%s)" % (self.postgresql)
336
337 - def __str__(self):
338 """ 339 Informal string representation for class instance. 340 """ 341 return self.__repr__()
342
343 - def __cmp__(self, other):
344 """ 345 Definition of equals operator for this class. 346 Lists within this class are "unordered" for equality comparisons. 347 @param other: Other object to compare to. 348 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 349 """ 350 if other is None: 351 return 1 352 if self.postgresql != other.postgresql: 353 if self.postgresql < other.postgresql: 354 return -1 355 else: 356 return 1 357 return 0
358
359 - def _setPostgresql(self, value):
360 """ 361 Property target used to set the postgresql configuration value. 362 If not C{None}, the value must be a C{PostgresqlConfig} object. 363 @raise ValueError: If the value is not a C{PostgresqlConfig} 364 """ 365 if value is None: 366 self._postgresql = None 367 else: 368 if not isinstance(value, PostgresqlConfig): 369 raise ValueError("Value must be a C{PostgresqlConfig} object.") 370 self._postgresql = value
371
372 - def _getPostgresql(self):
373 """ 374 Property target used to get the postgresql configuration value. 375 """ 376 return self._postgresql
377 378 postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.") 379
380 - def validate(self):
381 """ 382 Validates configuration represented by the object. 383 384 The compress mode must be filled in. Then, if the 'all' flag 385 I{is} set, no databases are allowed, and if the 'all' flag is 386 I{not} set, at least one database is required. 387 388 @raise ValueError: If one of the validations fails. 389 """ 390 if self.postgresql is None: 391 raise ValueError("PostgreSQL section is required.") 392 if self.postgresql.compressMode is None: 393 raise ValueError("Compress mode value is required.") 394 if self.postgresql.all: 395 if self.postgresql.databases is not None and self.postgresql.databases != []: 396 raise ValueError("Databases cannot be specified if 'all' flag is set.") 397 else: 398 if self.postgresql.databases is None or len(self.postgresql.databases) < 1: 399 raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
400
401 - def addConfig(self, xmlDom, parentNode):
402 """ 403 Adds a <postgresql> configuration section as the next child of a parent. 404 405 Third parties should use this function to write configuration related to 406 this extension. 407 408 We add the following fields to the document:: 409 410 user //cb_config/postgresql/user 411 compressMode //cb_config/postgresql/compress_mode 412 all //cb_config/postgresql/all 413 414 We also add groups of the following items, one list element per 415 item:: 416 417 database //cb_config/postgresql/database 418 419 @param xmlDom: DOM tree as from C{impl.createDocument()}. 420 @param parentNode: Parent that the section should be appended to. 421 """ 422 if self.postgresql is not None: 423 sectionNode = addContainerNode(xmlDom, parentNode, "postgresql") 424 addStringNode(xmlDom, sectionNode, "user", self.postgresql.user) 425 addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode) 426 addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all) 427 if self.postgresql.databases is not None: 428 for database in self.postgresql.databases: 429 addStringNode(xmlDom, sectionNode, "database", database)
430
431 - def _parseXmlData(self, xmlData):
432 """ 433 Internal method to parse an XML string into the object. 434 435 This method parses the XML document into a DOM tree (C{xmlDom}) and then 436 calls a static method to parse the postgresql configuration section. 437 438 @param xmlData: XML data to be parsed 439 @type xmlData: String data 440 441 @raise ValueError: If the XML cannot be successfully parsed. 442 """ 443 (xmlDom, parentNode) = createInputDom(xmlData) 444 self._postgresql = LocalConfig._parsePostgresql(parentNode)
445 446 @staticmethod
447 - def _parsePostgresql(parent):
448 """ 449 Parses a postgresql configuration section. 450 451 We read the following fields:: 452 453 user //cb_config/postgresql/user 454 compressMode //cb_config/postgresql/compress_mode 455 all //cb_config/postgresql/all 456 457 We also read groups of the following item, one list element per 458 item:: 459 460 databases //cb_config/postgresql/database 461 462 @param parent: Parent node to search beneath. 463 464 @return: C{PostgresqlConfig} object or C{None} if the section does not exist. 465 @raise ValueError: If some filled-in value is invalid. 466 """ 467 postgresql = None 468 section = readFirstChild(parent, "postgresql") 469 if section is not None: 470 postgresql = PostgresqlConfig() 471 postgresql.user = readString(section, "user") 472 postgresql.compressMode = readString(section, "compress_mode") 473 postgresql.all = readBoolean(section, "all") 474 postgresql.databases = readStringList(section, "database") 475 return postgresql
476
477 478 ######################################################################## 479 # Public functions 480 ######################################################################## 481 482 ########################### 483 # executeAction() function 484 ########################### 485 486 -def executeAction(configPath, options, config):
487 """ 488 Executes the PostgreSQL backup action. 489 490 @param configPath: Path to configuration file on disk. 491 @type configPath: String representing a path on disk. 492 493 @param options: Program command-line options. 494 @type options: Options object. 495 496 @param config: Program configuration. 497 @type config: Config object. 498 499 @raise ValueError: Under many generic error conditions 500 @raise IOError: If a backup could not be written for some reason. 501 """ 502 logger.debug("Executing PostgreSQL extended action.") 503 if config.options is None or config.collect is None: 504 raise ValueError("Cedar Backup configuration is not properly filled in.") 505 local = LocalConfig(xmlPath=configPath) 506 if local.postgresql.all: 507 logger.info("Backing up all databases.") 508 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user, 509 config.options.backupUser, config.options.backupGroup, None) 510 if local.postgresql.databases is not None and local.postgresql.databases != []: 511 logger.debug("Backing up %d individual databases.", len(local.postgresql.databases)) 512 for database in local.postgresql.databases: 513 logger.info("Backing up database [%s].", database) 514 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user, 515 config.options.backupUser, config.options.backupGroup, database) 516 logger.info("Executed the PostgreSQL extended action successfully.")
517
518 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
519 """ 520 Backs up an individual PostgreSQL database, or all databases. 521 522 This internal method wraps the public method and adds some functionality, 523 like figuring out a filename, etc. 524 525 @param targetDir: Directory into which backups should be written. 526 @param compressMode: Compress mode to be used for backed-up files. 527 @param user: User to use for connecting to the database. 528 @param backupUser: User to own resulting file. 529 @param backupGroup: Group to own resulting file. 530 @param database: Name of database, or C{None} for all databases. 531 532 @return: Name of the generated backup file. 533 534 @raise ValueError: If some value is missing or invalid. 535 @raise IOError: If there is a problem executing the PostgreSQL dump. 536 """ 537 (outputFile, filename) = _getOutputFile(targetDir, database, compressMode) 538 try: 539 backupDatabase(user, outputFile, database) 540 finally: 541 outputFile.close() 542 if not os.path.exists(filename): 543 raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename) 544 changeOwnership(filename, backupUser, backupGroup)
545
546 # pylint: disable=R0204 547 -def _getOutputFile(targetDir, database, compressMode):
548 """ 549 Opens the output file used for saving the PostgreSQL dump. 550 551 The filename is either C{"postgresqldump.txt"} or 552 C{"postgresqldump-<database>.txt"}. The C{".gz"} or C{".bz2"} extension is 553 added if C{compress} is C{True}. 554 555 @param targetDir: Target directory to write file in. 556 @param database: Name of the database (if any) 557 @param compressMode: Compress mode to be used for backed-up files. 558 559 @return: Tuple of (Output file object, filename) 560 """ 561 if database is None: 562 filename = os.path.join(targetDir, "postgresqldump.txt") 563 else: 564 filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database) 565 if compressMode == "gzip": 566 filename = "%s.gz" % filename 567 outputFile = GzipFile(filename, "w") 568 elif compressMode == "bzip2": 569 filename = "%s.bz2" % filename 570 outputFile = BZ2File(filename, "w") 571 else: 572 outputFile = open(filename, "w") 573 logger.debug("PostgreSQL dump file will be [%s].", filename) 574 return (outputFile, filename)
575
576 577 ############################ 578 # backupDatabase() function 579 ############################ 580 581 -def backupDatabase(user, backupFile, database=None):
582 """ 583 Backs up an individual PostgreSQL database, or all databases. 584 585 This function backs up either a named local PostgreSQL database or all local 586 PostgreSQL databases, using the passed in user for connectivity. 587 This is I{always} a full backup. There is no facility for incremental 588 backups. 589 590 The backup data will be written into the passed-in back file. Normally, 591 this would be an object as returned from C{open()}, but it is possible to 592 use something like a C{GzipFile} to write compressed output. The caller is 593 responsible for closing the passed-in backup file. 594 595 @note: Typically, you would use the C{root} user to back up all databases. 596 597 @param user: User to use for connecting to the database. 598 @type user: String representing PostgreSQL username. 599 600 @param backupFile: File use for writing backup. 601 @type backupFile: Python file object as from C{open()} or C{file()}. 602 603 @param database: Name of the database to be backed up. 604 @type database: String representing database name, or C{None} for all databases. 605 606 @raise ValueError: If some value is missing or invalid. 607 @raise IOError: If there is a problem executing the PostgreSQL dump. 608 """ 609 args = [] 610 if user is not None: 611 args.append('-U') 612 args.append(user) 613 614 if database is None: 615 command = resolveCommand(POSTGRESQLDUMPALL_COMMAND) 616 else: 617 command = resolveCommand(POSTGRESQLDUMP_COMMAND) 618 args.append(database) 619 620 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0] 621 if result != 0: 622 if database is None: 623 raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result) 624 else: 625 raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
626