ganeti-noded 17.7 KB
Newer Older
Iustin Pop's avatar
Iustin Pop committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#!/usr/bin/python
#

# Copyright (C) 2006, 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.


"""Ganeti node daemon"""

24
25
26
# functions in this module need to have a given name structure, so:
# pylint: disable-msg=C0103

Iustin Pop's avatar
Iustin Pop committed
27
28
29
import os
import sys
import traceback
Guido Trotter's avatar
Guido Trotter committed
30
import SocketServer
31
import errno
Iustin Pop's avatar
Iustin Pop committed
32
import logging
33
import signal
Iustin Pop's avatar
Iustin Pop committed
34
35
36
37
38
39
40
41

from optparse import OptionParser

from ganeti import backend
from ganeti import logger
from ganeti import constants
from ganeti import objects
from ganeti import errors
42
from ganeti import jstore
Iustin Pop's avatar
Iustin Pop committed
43
from ganeti import ssconf
44
from ganeti import http
45
from ganeti import utils
Iustin Pop's avatar
Iustin Pop committed
46
47


48
49
50
queue_lock = None


51
52
53
54
55
56
def _RequireJobQueueLock(fn):
  """Decorator for job queue manipulating functions.

  """
  def wrapper(*args, **kwargs):
    # Locking in exclusive, blocking mode because there could be several
57
58
    # children running at the same time. Waiting up to 10 seconds.
    queue_lock.Exclusive(blocking=True, timeout=10)
59
60
61
62
63
64
65
    try:
      return fn(*args, **kwargs)
    finally:
      queue_lock.Unlock()
  return wrapper


66
class NodeDaemonRequestHandler(http.HTTPRequestHandler):
67
68
69
70
71
  """The server implementation.

  This class holds all methods exposed over the RPC interface.

  """
72
73
  def HandleRequest(self):
    """Handle a request.
Iustin Pop's avatar
Iustin Pop committed
74

75
    """
76
77
78
    if self.command.upper() != "PUT":
      raise http.HTTPBadRequest()

Iustin Pop's avatar
Iustin Pop committed
79
80
81
82
    path = self.path
    if path.startswith("/"):
      path = path[1:]

83
84
85
    method = getattr(self, "perspective_%s" % path, None)
    if method is None:
      raise httperror.HTTPNotFound()
Iustin Pop's avatar
Iustin Pop committed
86

Iustin Pop's avatar
Iustin Pop committed
87
    try:
88
89
90
91
92
      try:
        return method(self.post_data)
      except:
        logging.exception("Error in RPC call")
        raise
93
    except errors.QuitGanetiException, err:
94
95
      # Tell parent to quit
      os.kill(self.server.noded_pid, signal.SIGTERM)
Iustin Pop's avatar
Iustin Pop committed
96
97
98

  # the new block devices  --------------------------

99
100
101
102
103
  @staticmethod
  def perspective_blockdev_create(params):
    """Create a block device.

    """
104
    bdev_s, size, owner, on_primary, info = params
105
    bdev = objects.Disk.FromDict(bdev_s)
Iustin Pop's avatar
Iustin Pop committed
106
107
    if bdev is None:
      raise ValueError("can't unserialize data!")
108
    return backend.CreateBlockDevice(bdev, size, owner, on_primary, info)
Iustin Pop's avatar
Iustin Pop committed
109

110
111
112
113
114
  @staticmethod
  def perspective_blockdev_remove(params):
    """Remove a block device.

    """
Iustin Pop's avatar
Iustin Pop committed
115
    bdev_s = params[0]
116
    bdev = objects.Disk.FromDict(bdev_s)
Iustin Pop's avatar
Iustin Pop committed
117
118
    return backend.RemoveBlockDevice(bdev)

Iustin Pop's avatar
Iustin Pop committed
119
120
121
122
123
124
125
126
  @staticmethod
  def perspective_blockdev_rename(params):
    """Remove a block device.

    """
    devlist = [(objects.Disk.FromDict(ds), uid) for ds, uid in params]
    return backend.RenameBlockDevices(devlist)

127
128
129
130
131
  @staticmethod
  def perspective_blockdev_assemble(params):
    """Assemble a block device.

    """
132
    bdev_s, owner, on_primary = params
133
    bdev = objects.Disk.FromDict(bdev_s)
Iustin Pop's avatar
Iustin Pop committed
134
135
    if bdev is None:
      raise ValueError("can't unserialize data!")
136
    return backend.AssembleBlockDevice(bdev, owner, on_primary)
Iustin Pop's avatar
Iustin Pop committed
137

138
139
140
141
142
  @staticmethod
  def perspective_blockdev_shutdown(params):
    """Shutdown a block device.

    """
Iustin Pop's avatar
Iustin Pop committed
143
    bdev_s = params[0]
144
    bdev = objects.Disk.FromDict(bdev_s)
Iustin Pop's avatar
Iustin Pop committed
145
146
147
148
    if bdev is None:
      raise ValueError("can't unserialize data!")
    return backend.ShutdownBlockDevice(bdev)

149
  @staticmethod
150
  def perspective_blockdev_addchildren(params):
151
152
153
154
155
156
    """Add a child to a mirror device.

    Note: this is only valid for mirror devices. It's the caller's duty
    to send a correct disk, otherwise we raise an error.

    """
Iustin Pop's avatar
Iustin Pop committed
157
    bdev_s, ndev_s = params
158
    bdev = objects.Disk.FromDict(bdev_s)
159
160
    ndevs = [objects.Disk.FromDict(disk_s) for disk_s in ndev_s]
    if bdev is None or ndevs.count(None) > 0:
Iustin Pop's avatar
Iustin Pop committed
161
      raise ValueError("can't unserialize data!")
162
    return backend.MirrorAddChildren(bdev, ndevs)
Iustin Pop's avatar
Iustin Pop committed
163

164
  @staticmethod
165
  def perspective_blockdev_removechildren(params):
166
167
168
169
170
171
    """Remove a child from a mirror device.

    This is only valid for mirror devices, of course. It's the callers
    duty to send a correct disk, otherwise we raise an error.

    """
Iustin Pop's avatar
Iustin Pop committed
172
    bdev_s, ndev_s = params
173
    bdev = objects.Disk.FromDict(bdev_s)
174
175
    ndevs = [objects.Disk.FromDict(disk_s) for disk_s in ndev_s]
    if bdev is None or ndevs.count(None) > 0:
Iustin Pop's avatar
Iustin Pop committed
176
      raise ValueError("can't unserialize data!")
177
    return backend.MirrorRemoveChildren(bdev, ndevs)
Iustin Pop's avatar
Iustin Pop committed
178

179
180
181
182
183
  @staticmethod
  def perspective_blockdev_getmirrorstatus(params):
    """Return the mirror status for a list of disks.

    """
184
    disks = [objects.Disk.FromDict(dsk_s)
Iustin Pop's avatar
Iustin Pop committed
185
186
187
            for dsk_s in params]
    return backend.GetMirrorStatus(disks)

188
189
190
191
192
193
194
  @staticmethod
  def perspective_blockdev_find(params):
    """Expose the FindBlockDevice functionality for a disk.

    This will try to find but not activate a disk.

    """
195
    disk = objects.Disk.FromDict(params[0])
Iustin Pop's avatar
Iustin Pop committed
196
197
    return backend.FindBlockDevice(disk)

198
199
200
201
202
203
204
205
206
  @staticmethod
  def perspective_blockdev_snapshot(params):
    """Create a snapshot device.

    Note that this is only valid for LVM disks, if we get passed
    something else we raise an exception. The snapshot device can be
    remove by calling the generic block device remove call.

    """
207
    cfbd = objects.Disk.FromDict(params[0])
Iustin Pop's avatar
Iustin Pop committed
208
209
    return backend.SnapshotBlockDevice(cfbd)

210
211
212
213
214
215
216
217
218
  @staticmethod
  def perspective_blockdev_grow(params):
    """Grow a stack of devices.

    """
    cfbd = objects.Disk.FromDict(params[0])
    amount = params[1]
    return backend.GrowBlockDevice(cfbd, amount)

219
220
221
222
223
224
225
226
  @staticmethod
  def perspective_blockdev_close(params):
    """Closes the given block devices.

    """
    disks = [objects.Disk.FromDict(cf) for cf in params]
    return backend.CloseBlockDevices(disks)

Iustin Pop's avatar
Iustin Pop committed
227
228
  # export/import  --------------------------

229
230
231
232
233
  @staticmethod
  def perspective_snapshot_export(params):
    """Export a given snapshot.

    """
234
    disk = objects.Disk.FromDict(params[0])
Iustin Pop's avatar
Iustin Pop committed
235
    dest_node = params[1]
236
    instance = objects.Instance.FromDict(params[2])
237
238
239
240
241
    return backend.ExportSnapshot(disk, dest_node, instance)

  @staticmethod
  def perspective_finalize_export(params):
    """Expose the finalize export functionality.
Iustin Pop's avatar
Iustin Pop committed
242

243
    """
244
245
    instance = objects.Instance.FromDict(params[0])
    snap_disks = [objects.Disk.FromDict(str_data)
Iustin Pop's avatar
Iustin Pop committed
246
247
248
                  for str_data in params[1]]
    return backend.FinalizeExport(instance, snap_disks)

249
250
251
252
253
254
255
256
257
258
  @staticmethod
  def perspective_export_info(params):
    """Query information about an existing export on this node.

    The given path may not contain an export, in which case we return
    None.

    """
    path = params[0]
    einfo = backend.ExportInfo(path)
Iustin Pop's avatar
Iustin Pop committed
259
260
261
262
    if einfo is None:
      return einfo
    return einfo.Dumps()

263
264
265
266
267
268
269
270
271
  @staticmethod
  def perspective_export_list(params):
    """List the available exports on this node.

    Note that as opposed to export_info, which may query data about an
    export in any path, this only queries the standard Ganeti path
    (constants.EXPORT_DIR).

    """
Iustin Pop's avatar
Iustin Pop committed
272
273
    return backend.ListExports()

274
275
276
277
278
  @staticmethod
  def perspective_export_remove(params):
    """Remove an export.

    """
Iustin Pop's avatar
Iustin Pop committed
279
280
281
282
283
    export = params[0]
    return backend.RemoveExport(export)

  # volume  --------------------------

284
285
286
287
288
  @staticmethod
  def perspective_volume_list(params):
    """Query the list of logical volumes in a given volume group.

    """
Iustin Pop's avatar
Iustin Pop committed
289
290
291
    vgname = params[0]
    return backend.GetVolumeList(vgname)

292
293
294
295
296
  @staticmethod
  def perspective_vg_list(params):
    """Query the list of volume groups.

    """
Iustin Pop's avatar
Iustin Pop committed
297
298
299
300
    return backend.ListVolumeGroups()

  # bridge  --------------------------

301
302
303
304
305
  @staticmethod
  def perspective_bridges_exist(params):
    """Check if all bridges given exist on this node.

    """
Iustin Pop's avatar
Iustin Pop committed
306
307
308
309
310
    bridges_list = params[0]
    return backend.BridgesExist(bridges_list)

  # instance  --------------------------

311
312
313
314
315
  @staticmethod
  def perspective_instance_os_add(params):
    """Install an OS on a given instance.

    """
Iustin Pop's avatar
Iustin Pop committed
316
    inst_s, os_disk, swap_disk = params
317
    inst = objects.Instance.FromDict(inst_s)
Iustin Pop's avatar
Iustin Pop committed
318
319
    return backend.AddOSToInstance(inst, os_disk, swap_disk)

320
321
322
323
324
325
  @staticmethod
  def perspective_instance_run_rename(params):
    """Runs the OS rename script for an instance.

    """
    inst_s, old_name, os_disk, swap_disk = params
326
    inst = objects.Instance.FromDict(inst_s)
327
328
    return backend.RunRenameInstance(inst, old_name, os_disk, swap_disk)

329
330
331
332
333
  @staticmethod
  def perspective_instance_os_import(params):
    """Run the import function of an OS onto a given instance.

    """
Iustin Pop's avatar
Iustin Pop committed
334
    inst_s, os_disk, swap_disk, src_node, src_image = params
335
    inst = objects.Instance.FromDict(inst_s)
Iustin Pop's avatar
Iustin Pop committed
336
337
338
    return backend.ImportOSIntoInstance(inst, os_disk, swap_disk,
                                        src_node, src_image)

339
340
341
342
343
  @staticmethod
  def perspective_instance_shutdown(params):
    """Shutdown an instance.

    """
344
    instance = objects.Instance.FromDict(params[0])
Iustin Pop's avatar
Iustin Pop committed
345
346
    return backend.ShutdownInstance(instance)

347
348
349
350
351
  @staticmethod
  def perspective_instance_start(params):
    """Start an instance.

    """
352
    instance = objects.Instance.FromDict(params[0])
Iustin Pop's avatar
Iustin Pop committed
353
354
355
    extra_args = params[1]
    return backend.StartInstance(instance, extra_args)

356
357
358
359
360
361
362
363
  @staticmethod
  def perspective_instance_migrate(params):
    """Migrates an instance.

    """
    instance, target, live = params
    return backend.MigrateInstance(instance, target, live)

364
365
366
367
368
369
370
371
372
373
  @staticmethod
  def perspective_instance_reboot(params):
    """Reboot an instance.

    """
    instance = objects.Instance.FromDict(params[0])
    reboot_type = params[1]
    extra_args = params[2]
    return backend.RebootInstance(instance, reboot_type, extra_args)

374
375
376
377
378
  @staticmethod
  def perspective_instance_info(params):
    """Query instance information.

    """
Iustin Pop's avatar
Iustin Pop committed
379
380
    return backend.GetInstanceInfo(params[0])

381
382
383
384
385
  @staticmethod
  def perspective_all_instances_info(params):
    """Query information about all instances.

    """
Iustin Pop's avatar
Iustin Pop committed
386
387
    return backend.GetAllInstancesInfo()

388
389
390
391
392
  @staticmethod
  def perspective_instance_list(params):
    """Query the list of running instances.

    """
Iustin Pop's avatar
Iustin Pop committed
393
394
395
396
    return backend.GetInstanceList()

  # node --------------------------

397
398
399
400
401
  @staticmethod
  def perspective_node_tcp_ping(params):
    """Do a TcpPing on the remote node.

    """
402
403
    return utils.TcpPing(params[1], params[2], timeout=params[3],
                         live_port_needed=params[4], source=params[0])
404

405
406
407
408
409
  @staticmethod
  def perspective_node_info(params):
    """Query node information.

    """
Iustin Pop's avatar
Iustin Pop committed
410
411
412
    vgname = params[0]
    return backend.GetNodeInfo(vgname)

413
414
415
416
417
  @staticmethod
  def perspective_node_add(params):
    """Complete the registration of this node in the cluster.

    """
Iustin Pop's avatar
Iustin Pop committed
418
419
420
    return backend.AddNode(params[0], params[1], params[2],
                           params[3], params[4], params[5])

421
422
423
424
425
  @staticmethod
  def perspective_node_verify(params):
    """Run a verify sequence on this node.

    """
Iustin Pop's avatar
Iustin Pop committed
426
427
    return backend.VerifyNode(params[0])

428
429
430
431
432
  @staticmethod
  def perspective_node_start_master(params):
    """Promote this node to master status.

    """
433
    return backend.StartMaster(params[0])
Iustin Pop's avatar
Iustin Pop committed
434

435
436
437
438
439
  @staticmethod
  def perspective_node_stop_master(params):
    """Demote this node from master status.

    """
440
    return backend.StopMaster(params[0])
Iustin Pop's avatar
Iustin Pop committed
441

442
443
444
445
446
  @staticmethod
  def perspective_node_leave_cluster(params):
    """Cleanup after leaving a cluster.

    """
Iustin Pop's avatar
Iustin Pop committed
447
448
    return backend.LeaveCluster()

449
450
451
452
453
  @staticmethod
  def perspective_node_volumes(params):
    """Query the list of all logical volume groups.

    """
454
455
    return backend.NodeVolumes()

Iustin Pop's avatar
Iustin Pop committed
456
457
  # cluster --------------------------

458
459
460
461
462
  @staticmethod
  def perspective_version(params):
    """Query version information.

    """
Iustin Pop's avatar
Iustin Pop committed
463
464
    return constants.PROTOCOL_VERSION

465
466
467
468
469
470
471
472
  @staticmethod
  def perspective_upload_file(params):
    """Upload a file.

    Note that the backend implementation imposes strict rules on which
    files are accepted.

    """
Iustin Pop's avatar
Iustin Pop committed
473
474
475
476
477
    return backend.UploadFile(*params)


  # os -----------------------

478
479
480
481
482
  @staticmethod
  def perspective_os_diagnose(params):
    """Query detailed information about existing OSes.

    """
483
    return [os.ToDict() for os in backend.DiagnoseOS()]
Iustin Pop's avatar
Iustin Pop committed
484

485
486
487
488
489
  @staticmethod
  def perspective_os_get(params):
    """Query information about a given OS.

    """
Iustin Pop's avatar
Iustin Pop committed
490
491
    name = params[0]
    try:
492
      os_obj = backend.OSFromDisk(name)
Iustin Pop's avatar
Iustin Pop committed
493
    except errors.InvalidOS, err:
494
495
      os_obj = objects.OS.FromInvalidOS(err)
    return os_obj.ToDict()
Iustin Pop's avatar
Iustin Pop committed
496
497
498

  # hooks -----------------------

499
500
501
502
503
  @staticmethod
  def perspective_hooks_runner(params):
    """Run hook scripts.

    """
Iustin Pop's avatar
Iustin Pop committed
504
505
506
507
    hpath, phase, env = params
    hr = backend.HooksRunner()
    return hr.RunHooks(hpath, phase, env)

508
509
510
511
512
513
514
515
516
517
518
  # iallocator -----------------

  @staticmethod
  def perspective_iallocator_runner(params):
    """Run an iallocator script.

    """
    name, idata = params
    iar = backend.IAllocatorRunner()
    return iar.Run(name, idata)

519
520
521
522
523
524
525
526
527
528
  # test -----------------------

  @staticmethod
  def perspective_test_delay(params):
    """Run test delay.

    """
    duration = params[0]
    return utils.TestDelay(duration)

529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
  @staticmethod
  def perspective_file_storage_dir_create(params):
    """Create the file storage directory.

    """
    file_storage_dir = params[0]
    return backend.CreateFileStorageDir(file_storage_dir)

  @staticmethod
  def perspective_file_storage_dir_remove(params):
    """Remove the file storage directory.

    """
    file_storage_dir = params[0]
    return backend.RemoveFileStorageDir(file_storage_dir)

  @staticmethod
  def perspective_file_storage_dir_rename(params):
    """Rename the file storage directory.

    """
    old_file_storage_dir = params[0]
    new_file_storage_dir = params[1]
    return backend.RenameFileStorageDir(old_file_storage_dir,
                                        new_file_storage_dir)

555
  @staticmethod
556
  @_RequireJobQueueLock
557
558
559
560
561
  def perspective_jobqueue_update(params):
    """Update job queue.

    """
    (file_name, content) = params
562
    return backend.JobQueueUpdate(file_name, content)
563
564

  @staticmethod
565
  @_RequireJobQueueLock
566
567
568
569
570
571
  def perspective_jobqueue_purge(params):
    """Purge job queue.

    """
    return backend.JobQueuePurge()

572
573
574
575
576
577
578
579
580
581
  @staticmethod
  @_RequireJobQueueLock
  def perspective_jobqueue_rename(params):
    """Rename a job queue file.

    """
    (old, new) = params

    return backend.JobQueueRename(old, new)

Iustin Pop's avatar
Iustin Pop committed
582

583
584
585
class NodeDaemonHttpServer(http.HTTPServer):
  def __init__(self, server_address):
    http.HTTPServer.__init__(self, server_address, NodeDaemonRequestHandler)
586
587
588
589
590
591
592
593
594
595
596
    self.noded_pid = os.getpid()

  def serve_forever(self):
    """Handle requests until told to quit."""
    sighandler = utils.SignalHandler([signal.SIGINT, signal.SIGTERM])
    try:
      while not sighandler.called:
        self.handle_request()
      # TODO: There could be children running at this point
    finally:
      sighandler.Reset()
597
598


Guido Trotter's avatar
Guido Trotter committed
599
600
601
602
603
604
605
606
607
class ForkingHTTPServer(SocketServer.ForkingMixIn, NodeDaemonHttpServer):
  """Forking HTTP Server.

  This inherits from ForkingMixIn and HTTPServer in order to fork for each
  request we handle. This allows more requests to be handled concurrently.

  """


Iustin Pop's avatar
Iustin Pop committed
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
def ParseOptions():
  """Parse the command line options.

  Returns:
    (options, args) as from OptionParser.parse_args()

  """
  parser = OptionParser(description="Ganeti node daemon",
                        usage="%prog [-f] [-d]",
                        version="%%prog (ganeti) %s" %
                        constants.RELEASE_VERSION)

  parser.add_option("-f", "--foreground", dest="fork",
                    help="Don't detach from the current terminal",
                    default=True, action="store_false")
  parser.add_option("-d", "--debug", dest="debug",
                    help="Enable some debug messages",
                    default=False, action="store_true")
  options, args = parser.parse_args()
  return options, args


def main():
631
632
633
  """Main function for the node daemon.

  """
634
635
  global queue_lock

Iustin Pop's avatar
Iustin Pop committed
636
  options, args = ParseOptions()
637
  utils.debug = options.debug
Iustin Pop's avatar
Iustin Pop committed
638
639
640
641
642
643
644
645
646
647
648
649
650
  for fname in (constants.SSL_CERT_FILE,):
    if not os.path.isfile(fname):
      print "config %s not there, will not run." % fname
      sys.exit(5)

  try:
    ss = ssconf.SimpleStore()
    port = ss.GetNodeDaemonPort()
    pwdata = ss.GetNodeDaemonPassword()
  except errors.ConfigurationError, err:
    print "Cluster configuration incomplete: '%s'" % str(err)
    sys.exit(5)

651
652
653
654
655
656
657
658
659
660
661
662
663
664
  # create the various SUB_RUN_DIRS, if not existing, so that we handle the
  # situation where RUN_DIR is tmpfs
  for dir_name in constants.SUB_RUN_DIRS:
    if not os.path.exists(dir_name):
      try:
        os.mkdir(dir_name, 0755)
      except EnvironmentError, err:
        if err.errno != errno.EEXIST:
          print ("Node setup wrong, cannot create directory %s: %s" %
                 (dir_name, err))
          sys.exit(5)
    if not os.path.isdir(dir_name):
      print ("Node setup wrong, %s is not a directory" % dir_name)
      sys.exit(5)
665

Iustin Pop's avatar
Iustin Pop committed
666
667
  # become a daemon
  if options.fork:
668
    utils.Daemonize(logfile=constants.LOG_NODESERVER)
Iustin Pop's avatar
Iustin Pop committed
669

670
  utils.WritePidFile(constants.NODED_PID)
671

Iustin Pop's avatar
Iustin Pop committed
672
673
  logger.SetupLogging(logfile=constants.LOG_NODESERVER, debug=options.debug,
                      stderr_logging=not options.fork)
Iustin Pop's avatar
Iustin Pop committed
674
  logging.info("ganeti node daemon startup")
Iustin Pop's avatar
Iustin Pop committed
675

676
677
678
  # Prepare job queue
  queue_lock = jstore.InitAndVerifyQueue(must_lock=False)

Guido Trotter's avatar
Guido Trotter committed
679
  if options.fork:
680
    server = ForkingHTTPServer(('', port))
Guido Trotter's avatar
Guido Trotter committed
681
  else:
682
    server = NodeDaemonHttpServer(('', port))
Iustin Pop's avatar
Iustin Pop committed
683

684
685
686
  try:
    server.serve_forever()
  finally:
687
    utils.RemovePidFile(constants.NODED_PID)
688

Iustin Pop's avatar
Iustin Pop committed
689

690
if __name__ == '__main__':
Iustin Pop's avatar
Iustin Pop committed
691
  main()