1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42 """
43 Provides an extension to back up PostgreSQL databases.
44
45 This is a Cedar Backup extension used to back up PostgreSQL databases via the
46 Cedar Backup command line. It requires a new configurations section
47 <postgresql> and is intended to be run either immediately before or immediately
48 after the standard collect action. Aside from its own configuration, it
49 requires the options and collect configuration sections in the standard Cedar
50 Backup configuration file.
51
52 The backup is done via the C{pg_dump} or C{pg_dumpall} commands included with
53 the PostgreSQL product. Output can be compressed using C{gzip} or C{bzip2}.
54 Administrators can configure the extension either to back up all databases or
55 to back up only specific databases. The extension assumes that the current
56 user has passwordless access to the database since there is no easy way to pass
57 a password to the C{pg_dump} client. This can be accomplished using appropriate
58 voodoo in the C{pg_hda.conf} file.
59
60 Note that this code always produces a full backup. There is currently no
61 facility for making incremental backups.
62
63 You should always make C{/etc/cback.conf} unreadble to non-root users once you
64 place postgresql configuration into it, since postgresql configuration will
65 contain information about available PostgreSQL databases and usernames.
66
67 Use of this extension I{may} expose usernames in the process listing (via
68 C{ps}) when the backup is running if the username is specified in the
69 configuration.
70
71 @author: Kenneth J. Pronovici <pronovic@ieee.org>
72 @author: Antoine Beaupre <anarcat@koumbit.org>
73 """
74
75
76
77
78
79
80 import os
81 import logging
82 from gzip import GzipFile
83 from bz2 import BZ2File
84
85
86 from CedarBackup2.xmlutil import createInputDom, addContainerNode, addStringNode, addBooleanNode
87 from CedarBackup2.xmlutil import readFirstChild, readString, readStringList, readBoolean
88 from CedarBackup2.config import VALID_COMPRESS_MODES
89 from CedarBackup2.util import resolveCommand, executeCommand
90 from CedarBackup2.util import ObjectTypeList, changeOwnership
91
92
93
94
95
96
97 logger = logging.getLogger("CedarBackup2.log.extend.postgresql")
98 POSTGRESQLDUMP_COMMAND = [ "pg_dump", ]
99 POSTGRESQLDUMPALL_COMMAND = [ "pg_dumpall", ]
100
101
102
103
104
105
106 -class PostgresqlConfig(object):
107
108 """
109 Class representing PostgreSQL configuration.
110
111 The PostgreSQL configuration information is used for backing up PostgreSQL databases.
112
113 The following restrictions exist on data in this class:
114
115 - The compress mode must be one of the values in L{VALID_COMPRESS_MODES}.
116 - The 'all' flag must be 'Y' if no databases are defined.
117 - The 'all' flag must be 'N' if any databases are defined.
118 - Any values in the databases list must be strings.
119
120 @sort: __init__, __repr__, __str__, __cmp__, user, all, databases
121 """
122
123 - def __init__(self, user=None, compressMode=None, all=None, databases=None):
124 """
125 Constructor for the C{PostgresqlConfig} class.
126
127 @param user: User to execute backup as.
128 @param compressMode: Compress mode for backed-up files.
129 @param all: Indicates whether to back up all databases.
130 @param databases: List of databases to back up.
131 """
132 self._user = None
133 self._compressMode = None
134 self._all = None
135 self._databases = None
136 self.user = user
137 self.compressMode = compressMode
138 self.all = all
139 self.databases = databases
140
141 - def __repr__(self):
142 """
143 Official string representation for class instance.
144 """
145 return "PostgresqlConfig(%s, %s, %s)" % (self.user, self.all, self.databases)
146
148 """
149 Informal string representation for class instance.
150 """
151 return self.__repr__()
152
153 - def __cmp__(self, other):
154 """
155 Definition of equals operator for this class.
156 @param other: Other object to compare to.
157 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
158 """
159 if other is None:
160 return 1
161 if self.user != other.user:
162 if self.user < other.user:
163 return -1
164 else:
165 return 1
166 if self.compressMode != other.compressMode:
167 if self.compressMode < other.compressMode:
168 return -1
169 else:
170 return 1
171 if self.all != other.all:
172 if self.all < other.all:
173 return -1
174 else:
175 return 1
176 if self.databases != other.databases:
177 if self.databases < other.databases:
178 return -1
179 else:
180 return 1
181 return 0
182
183 - def _setUser(self, value):
184 """
185 Property target used to set the user value.
186 """
187 if value is not None:
188 if len(value) < 1:
189 raise ValueError("User must be non-empty string.")
190 self._user = value
191
192 - def _getUser(self):
193 """
194 Property target used to get the user value.
195 """
196 return self._user
197
198 - def _setCompressMode(self, value):
199 """
200 Property target used to set the compress mode.
201 If not C{None}, the mode must be one of the values in L{VALID_COMPRESS_MODES}.
202 @raise ValueError: If the value is not valid.
203 """
204 if value is not None:
205 if value not in VALID_COMPRESS_MODES:
206 raise ValueError("Compress mode must be one of %s." % VALID_COMPRESS_MODES)
207 self._compressMode = value
208
210 """
211 Property target used to get the compress mode.
212 """
213 return self._compressMode
214
215 - def _setAll(self, value):
216 """
217 Property target used to set the 'all' flag.
218 No validations, but we normalize the value to C{True} or C{False}.
219 """
220 if value:
221 self._all = True
222 else:
223 self._all = False
224
226 """
227 Property target used to get the 'all' flag.
228 """
229 return self._all
230
231 - def _setDatabases(self, value):
232 """
233 Property target used to set the databases list.
234 Either the value must be C{None} or each element must be a string.
235 @raise ValueError: If the value is not a string.
236 """
237 if value is None:
238 self._databases = None
239 else:
240 for database in value:
241 if len(database) < 1:
242 raise ValueError("Each database must be a non-empty string.")
243 try:
244 saved = self._databases
245 self._databases = ObjectTypeList(basestring, "string")
246 self._databases.extend(value)
247 except Exception, e:
248 self._databases = saved
249 raise e
250
251 - def _getDatabases(self):
252 """
253 Property target used to get the databases list.
254 """
255 return self._databases
256
257 user = property(_getUser, _setUser, None, "User to execute backup as.")
258 compressMode = property(_getCompressMode, _setCompressMode, None, "Compress mode to be used for backed-up files.")
259 all = property(_getAll, _setAll, None, "Indicates whether to back up all databases.")
260 databases = property(_getDatabases, _setDatabases, None, "List of databases to back up.")
261
268
269 """
270 Class representing this extension's configuration document.
271
272 This is not a general-purpose configuration object like the main Cedar
273 Backup configuration object. Instead, it just knows how to parse and emit
274 PostgreSQL-specific configuration values. Third parties who need to read and
275 write configuration related to this extension should access it through the
276 constructor, C{validate} and C{addConfig} methods.
277
278 @note: Lists within this class are "unordered" for equality comparisons.
279
280 @sort: __init__, __repr__, __str__, __cmp__, postgresql, validate, addConfig
281 """
282
283 - def __init__(self, xmlData=None, xmlPath=None, validate=True):
284 """
285 Initializes a configuration object.
286
287 If you initialize the object without passing either C{xmlData} or
288 C{xmlPath} then configuration will be empty and will be invalid until it
289 is filled in properly.
290
291 No reference to the original XML data or original path is saved off by
292 this class. Once the data has been parsed (successfully or not) this
293 original information is discarded.
294
295 Unless the C{validate} argument is C{False}, the L{LocalConfig.validate}
296 method will be called (with its default arguments) against configuration
297 after successfully parsing any passed-in XML. Keep in mind that even if
298 C{validate} is C{False}, it might not be possible to parse the passed-in
299 XML document if lower-level validations fail.
300
301 @note: It is strongly suggested that the C{validate} option always be set
302 to C{True} (the default) unless there is a specific need to read in
303 invalid configuration from disk.
304
305 @param xmlData: XML data representing configuration.
306 @type xmlData: String data.
307
308 @param xmlPath: Path to an XML file on disk.
309 @type xmlPath: Absolute path to a file on disk.
310
311 @param validate: Validate the document after parsing it.
312 @type validate: Boolean true/false.
313
314 @raise ValueError: If both C{xmlData} and C{xmlPath} are passed-in.
315 @raise ValueError: If the XML data in C{xmlData} or C{xmlPath} cannot be parsed.
316 @raise ValueError: If the parsed configuration document is not valid.
317 """
318 self._postgresql = None
319 self.postgresql = None
320 if xmlData is not None and xmlPath is not None:
321 raise ValueError("Use either xmlData or xmlPath, but not both.")
322 if xmlData is not None:
323 self._parseXmlData(xmlData)
324 if validate:
325 self.validate()
326 elif xmlPath is not None:
327 xmlData = open(xmlPath).read()
328 self._parseXmlData(xmlData)
329 if validate:
330 self.validate()
331
333 """
334 Official string representation for class instance.
335 """
336 return "LocalConfig(%s)" % (self.postgresql)
337
339 """
340 Informal string representation for class instance.
341 """
342 return self.__repr__()
343
345 """
346 Definition of equals operator for this class.
347 Lists within this class are "unordered" for equality comparisons.
348 @param other: Other object to compare to.
349 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
350 """
351 if other is None:
352 return 1
353 if self.postgresql != other.postgresql:
354 if self.postgresql < other.postgresql:
355 return -1
356 else:
357 return 1
358 return 0
359
360 - def _setPostgresql(self, value):
361 """
362 Property target used to set the postgresql configuration value.
363 If not C{None}, the value must be a C{PostgresqlConfig} object.
364 @raise ValueError: If the value is not a C{PostgresqlConfig}
365 """
366 if value is None:
367 self._postgresql = None
368 else:
369 if not isinstance(value, PostgresqlConfig):
370 raise ValueError("Value must be a C{PostgresqlConfig} object.")
371 self._postgresql = value
372
373 - def _getPostgresql(self):
374 """
375 Property target used to get the postgresql configuration value.
376 """
377 return self._postgresql
378
379 postgresql = property(_getPostgresql, _setPostgresql, None, "Postgresql configuration in terms of a C{PostgresqlConfig} object.")
380
382 """
383 Validates configuration represented by the object.
384
385 The compress mode must be filled in. Then, if the 'all' flag
386 I{is} set, no databases are allowed, and if the 'all' flag is
387 I{not} set, at least one database is required.
388
389 @raise ValueError: If one of the validations fails.
390 """
391 if self.postgresql is None:
392 raise ValueError("PostgreSQL section is required.")
393 if self.postgresql.compressMode is None:
394 raise ValueError("Compress mode value is required.")
395 if self.postgresql.all:
396 if self.postgresql.databases is not None and self.postgresql.databases != []:
397 raise ValueError("Databases cannot be specified if 'all' flag is set.")
398 else:
399 if self.postgresql.databases is None or len(self.postgresql.databases) < 1:
400 raise ValueError("At least one PostgreSQL database must be indicated if 'all' flag is not set.")
401
403 """
404 Adds a <postgresql> configuration section as the next child of a parent.
405
406 Third parties should use this function to write configuration related to
407 this extension.
408
409 We add the following fields to the document::
410
411 user //cb_config/postgresql/user
412 compressMode //cb_config/postgresql/compress_mode
413 all //cb_config/postgresql/all
414
415 We also add groups of the following items, one list element per
416 item::
417
418 database //cb_config/postgresql/database
419
420 @param xmlDom: DOM tree as from C{impl.createDocument()}.
421 @param parentNode: Parent that the section should be appended to.
422 """
423 if self.postgresql is not None:
424 sectionNode = addContainerNode(xmlDom, parentNode, "postgresql")
425 addStringNode(xmlDom, sectionNode, "user", self.postgresql.user)
426 addStringNode(xmlDom, sectionNode, "compress_mode", self.postgresql.compressMode)
427 addBooleanNode(xmlDom, sectionNode, "all", self.postgresql.all)
428 if self.postgresql.databases is not None:
429 for database in self.postgresql.databases:
430 addStringNode(xmlDom, sectionNode, "database", database)
431
433 """
434 Internal method to parse an XML string into the object.
435
436 This method parses the XML document into a DOM tree (C{xmlDom}) and then
437 calls a static method to parse the postgresql configuration section.
438
439 @param xmlData: XML data to be parsed
440 @type xmlData: String data
441
442 @raise ValueError: If the XML cannot be successfully parsed.
443 """
444 (xmlDom, parentNode) = createInputDom(xmlData)
445 self._postgresql = LocalConfig._parsePostgresql(parentNode)
446
447 @staticmethod
448 - def _parsePostgresql(parent):
449 """
450 Parses a postgresql configuration section.
451
452 We read the following fields::
453
454 user //cb_config/postgresql/user
455 compressMode //cb_config/postgresql/compress_mode
456 all //cb_config/postgresql/all
457
458 We also read groups of the following item, one list element per
459 item::
460
461 databases //cb_config/postgresql/database
462
463 @param parent: Parent node to search beneath.
464
465 @return: C{PostgresqlConfig} object or C{None} if the section does not exist.
466 @raise ValueError: If some filled-in value is invalid.
467 """
468 postgresql = None
469 section = readFirstChild(parent, "postgresql")
470 if section is not None:
471 postgresql = PostgresqlConfig()
472 postgresql.user = readString(section, "user")
473 postgresql.compressMode = readString(section, "compress_mode")
474 postgresql.all = readBoolean(section, "all")
475 postgresql.databases = readStringList(section, "database")
476 return postgresql
477
478
479
480
481
482
483
484
485
486
487 -def executeAction(configPath, options, config):
488 """
489 Executes the PostgreSQL backup action.
490
491 @param configPath: Path to configuration file on disk.
492 @type configPath: String representing a path on disk.
493
494 @param options: Program command-line options.
495 @type options: Options object.
496
497 @param config: Program configuration.
498 @type config: Config object.
499
500 @raise ValueError: Under many generic error conditions
501 @raise IOError: If a backup could not be written for some reason.
502 """
503 logger.debug("Executing PostgreSQL extended action.")
504 if config.options is None or config.collect is None:
505 raise ValueError("Cedar Backup configuration is not properly filled in.")
506 local = LocalConfig(xmlPath=configPath)
507 if local.postgresql.all:
508 logger.info("Backing up all databases.")
509 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
510 config.options.backupUser, config.options.backupGroup, None)
511 if local.postgresql.databases is not None and local.postgresql.databases != []:
512 logger.debug("Backing up %d individual databases." % len(local.postgresql.databases))
513 for database in local.postgresql.databases:
514 logger.info("Backing up database [%s]." % database)
515 _backupDatabase(config.collect.targetDir, local.postgresql.compressMode, local.postgresql.user,
516 config.options.backupUser, config.options.backupGroup, database)
517 logger.info("Executed the PostgreSQL extended action successfully.")
518
519 -def _backupDatabase(targetDir, compressMode, user, backupUser, backupGroup, database=None):
520 """
521 Backs up an individual PostgreSQL database, or all databases.
522
523 This internal method wraps the public method and adds some functionality,
524 like figuring out a filename, etc.
525
526 @param targetDir: Directory into which backups should be written.
527 @param compressMode: Compress mode to be used for backed-up files.
528 @param user: User to use for connecting to the database.
529 @param backupUser: User to own resulting file.
530 @param backupGroup: Group to own resulting file.
531 @param database: Name of database, or C{None} for all databases.
532
533 @return: Name of the generated backup file.
534
535 @raise ValueError: If some value is missing or invalid.
536 @raise IOError: If there is a problem executing the PostgreSQL dump.
537 """
538 (outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
539 try:
540 backupDatabase(user, outputFile, database)
541 finally:
542 outputFile.close()
543 if not os.path.exists(filename):
544 raise IOError("Dump file [%s] does not seem to exist after backup completed." % filename)
545 changeOwnership(filename, backupUser, backupGroup)
546
548 """
549 Opens the output file used for saving the PostgreSQL dump.
550
551 The filename is either C{"postgresqldump.txt"} or
552 C{"postgresqldump-<database>.txt"}. The C{".gz"} or C{".bz2"} extension is
553 added if C{compress} is C{True}.
554
555 @param targetDir: Target directory to write file in.
556 @param database: Name of the database (if any)
557 @param compressMode: Compress mode to be used for backed-up files.
558
559 @return: Tuple of (Output file object, filename)
560 """
561 if database is None:
562 filename = os.path.join(targetDir, "postgresqldump.txt")
563 else:
564 filename = os.path.join(targetDir, "postgresqldump-%s.txt" % database)
565 if compressMode == "gzip":
566 filename = "%s.gz" % filename
567 outputFile = GzipFile(filename, "w")
568 elif compressMode == "bzip2":
569 filename = "%s.bz2" % filename
570 outputFile = BZ2File(filename, "w")
571 else:
572 outputFile = open(filename, "w")
573 logger.debug("PostgreSQL dump file will be [%s]." % filename)
574 return (outputFile, filename)
575
576
577
578
579
580
581 -def backupDatabase(user, backupFile, database=None):
582 """
583 Backs up an individual PostgreSQL database, or all databases.
584
585 This function backs up either a named local PostgreSQL database or all local
586 PostgreSQL databases, using the passed in user for connectivity.
587 This is I{always} a full backup. There is no facility for incremental
588 backups.
589
590 The backup data will be written into the passed-in back file. Normally,
591 this would be an object as returned from C{open()}, but it is possible to
592 use something like a C{GzipFile} to write compressed output. The caller is
593 responsible for closing the passed-in backup file.
594
595 @note: Typically, you would use the C{root} user to back up all databases.
596
597 @param user: User to use for connecting to the database.
598 @type user: String representing PostgreSQL username.
599
600 @param backupFile: File use for writing backup.
601 @type backupFile: Python file object as from C{open()} or C{file()}.
602
603 @param database: Name of the database to be backed up.
604 @type database: String representing database name, or C{None} for all databases.
605
606 @raise ValueError: If some value is missing or invalid.
607 @raise IOError: If there is a problem executing the PostgreSQL dump.
608 """
609 args = []
610 if user is not None:
611 args.append('-U')
612 args.append(user)
613
614 if database is None:
615 command = resolveCommand(POSTGRESQLDUMPALL_COMMAND)
616 else:
617 command = resolveCommand(POSTGRESQLDUMP_COMMAND)
618 args.append(database)
619
620 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True, doNotLog=True, outputFile=backupFile)[0]
621 if result != 0:
622 if database is None:
623 raise IOError("Error [%d] executing PostgreSQL database dump for all databases." % result)
624 else:
625 raise IOError("Error [%d] executing PostgreSQL database dump for database [%s]." % (result, database))
626