Coverage for drivers/util.py: 44%

414 elif "VDI_ACTIVATE" in caps or "VDI_DEACTIVATE" in caps: 414 ↛ 415line 414 didn't jump to line 415, because the condition on line 414 was never true

415 SMlog("Warning: vdi_[de]activate present for %s" % driver_info["name"])

416

417 results['capabilities'] = caps

418 # add in the configuration options

419 options = []

420 for option in driver_info['configuration']:

421 options.append({'key': option[0], 'description': option[1]})

422 results['configuration'] = options

423 return xmlrpc.client.dumps((results, ), "", True)

424

425

426def return_nil():

427 return xmlrpc.client.dumps((None, ), "", True, allow_none=True)

428

429

430def SRtoXML(SRlist):

431 dom = xml.dom.minidom.Document()

432 driver = dom.createElement("SRlist")

433 dom.appendChild(driver)

434

435 for key in SRlist.keys():

436 dict = SRlist[key]

437 entry = dom.createElement("SR")

438 driver.appendChild(entry)

439

440 e = dom.createElement("UUID")

441 entry.appendChild(e)

442 textnode = dom.createTextNode(key)

443 e.appendChild(textnode)

444

445 if 'size' in dict:

446 e = dom.createElement("Size")

447 entry.appendChild(e)

448 textnode = dom.createTextNode(str(dict['size']))

449 e.appendChild(textnode)

450

451 if 'storagepool' in dict:

452 e = dom.createElement("StoragePool")

453 entry.appendChild(e)

454 textnode = dom.createTextNode(str(dict['storagepool']))

455 e.appendChild(textnode)

456

457 if 'aggregate' in dict:

458 e = dom.createElement("Aggregate")

459 entry.appendChild(e)

460 textnode = dom.createTextNode(str(dict['aggregate']))

461 e.appendChild(textnode)

462

463 return dom.toprettyxml()

464

465

466def pathexists(path):

467 try:

468 os.lstat(path)

469 return True

470 except OSError as inst:

471 if inst.errno == errno.EIO: 471 ↛ 472line 471 didn't jump to line 472, because the condition on line 471 was never true

472 time.sleep(1)

473 try:

474 listdir(os.path.realpath(os.path.dirname(path)))

475 os.lstat(path)

476 return True

477 except:

478 pass

479 raise CommandException(errno.EIO, "os.lstat(%s)" % path, "failed")

480 return False

481

482

483def force_unlink(path):

484 try:

485 os.unlink(path)

486 except OSError as e:

487 if e.errno != errno.ENOENT: 487 ↛ 488line 487 didn't jump to line 488, because the condition on line 487 was never true

488 raise

489

490

491def create_secret(session, secret):

492 ref = session.xenapi.secret.create({'value': secret})

493 return session.xenapi.secret.get_uuid(ref)

494

495

496def get_secret(session, uuid):

497 try:

498 ref = session.xenapi.secret.get_by_uuid(uuid)

499 return session.xenapi.secret.get_value(ref)

500 except:

501 raise xs_errors.XenError('InvalidSecret', opterr='Unable to look up secret [%s]' % uuid)

502

503

504def get_real_path(path):

505 "Follow symlinks to the actual file"

506 absPath = path

507 directory = ''

508 while os.path.islink(absPath):

509 directory = os.path.dirname(absPath)

510 absPath = os.readlink(absPath)

511 absPath = os.path.join(directory, absPath)

512 return absPath

513

514

515def wait_for_path(path, timeout):

516 for i in range(0, timeout): 516 ↛ 520line 516 didn't jump to line 520, because the loop on line 516 didn't complete

517 if len(glob.glob(path)): 517 ↛ 519line 517 didn't jump to line 519, because the condition on line 517 was never false

518 return True

519 time.sleep(1)

520 return False

521

522

523def wait_for_nopath(path, timeout):

524 for i in range(0, timeout):

525 if not os.path.exists(path):

526 return True

527 time.sleep(1)

528 return False

529

530

531def wait_for_path_multi(path, timeout):

532 for i in range(0, timeout):

533 paths = glob.glob(path)

534 SMlog("_wait_for_paths_multi: paths = %s" % paths)

535 if len(paths):

536 SMlog("_wait_for_paths_multi: return first path: %s" % paths[0])

537 return paths[0]

538 time.sleep(1)

539 return ""

540

541

542def isdir(path):

543 try:

544 st = os.stat(path)

545 return stat.S_ISDIR(st.st_mode)

546 except OSError as inst:

547 if inst.errno == errno.EIO: 547 ↛ 548line 547 didn't jump to line 548, because the condition on line 547 was never true

548 raise CommandException(errno.EIO, "os.stat(%s)" % path, "failed")

549 return False

550

551

552def get_single_entry(path):

553 f = open(path, 'r')

554 line = f.readline()

555 f.close()

556 return line.rstrip()

557

558

559def get_fs_size(path):

560 st = ioretry_stat(path)

561 return st.f_blocks * st.f_frsize

562

563

564def get_fs_utilisation(path):

565 st = ioretry_stat(path)

566 return (st.f_blocks - st.f_bfree) * \

567 st.f_frsize

568

569

570def ismount(path):

571 """Test whether a path is a mount point"""

572 try:

573 s1 = os.stat(path)

574 s2 = os.stat(os.path.join(path, '..'))

575 except OSError as inst:

576 raise CommandException(inst.errno, "os.stat")

577 dev1 = s1.st_dev

578 dev2 = s2.st_dev

579 if dev1 != dev2:

580 return True # path/.. on a different device as path

581 ino1 = s1.st_ino

582 ino2 = s2.st_ino

583 if ino1 == ino2:

584 return True # path/.. is the same i-node as path

585 return False

586

587

588def makedirs(name, mode=0o777):

589 head, tail = os.path.split(name)

590 if not tail: 590 ↛ 591line 590 didn't jump to line 591, because the condition on line 590 was never true

591 head, tail = os.path.split(head)

592 if head and tail and not pathexists(head):

593 makedirs(head, mode)

594 if tail == os.curdir: 594 ↛ 595line 594 didn't jump to line 595, because the condition on line 594 was never true

595 return

596 try:

597 os.mkdir(name, mode)

598 except OSError as exc:

599 if exc.errno == errno.EEXIST and os.path.isdir(name): 599 ↛ 600line 599 didn't jump to line 600, because the condition on line 599 was never true

600 if mode:

601 os.chmod(name, mode)

602 pass

603 else:

604 raise

605

606

607def zeroOut(path, fromByte, bytes):

608 """write 'bytes' zeros to 'path' starting from fromByte (inclusive)"""

609 blockSize = 4096

610

611 fromBlock = fromByte // blockSize

612 if fromByte % blockSize:

613 fromBlock += 1

614 bytesBefore = fromBlock * blockSize - fromByte

615 if bytesBefore > bytes:

616 bytesBefore = bytes

617 bytes -= bytesBefore

618 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",

619 "seek=%s" % fromByte, "count=%s" % bytesBefore]

620 try:

621 pread2(cmd)

622 except CommandException:

623 return False

624

625 blocks = bytes // blockSize

626 bytes -= blocks * blockSize

627 fromByte = (fromBlock + blocks) * blockSize

628 if blocks:

629 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=%s" % blockSize,

630 "seek=%s" % fromBlock, "count=%s" % blocks]

631 try:

632 pread2(cmd)

633 except CommandException:

634 return False

635

636 if bytes:

637 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1",

638 "seek=%s" % fromByte, "count=%s" % bytes]

639 try:

640 pread2(cmd)

641 except CommandException:

642 return False

643

644 return True

645

646

647def wipefs(blockdev):

648 "Wipe filesystem signatures from `blockdev`"

649 pread2(["/usr/sbin/wipefs", "-a", blockdev])

650

651

652def match_rootdev(s):

653 regex = re.compile("^PRIMARY_DISK")

654 return regex.search(s, 0)

655

656

657def getrootdev():

658 filename = '/etc/xensource-inventory'

659 try:

660 f = open(filename, 'r')

661 except:

662 raise xs_errors.XenError('EIO', \

663 opterr="Unable to open inventory file [%s]" % filename)

664 rootdev = ''

665 for line in filter(match_rootdev, f.readlines()):

666 rootdev = line.split("'")[1]

667 if not rootdev: 667 ↛ 668line 667 didn't jump to line 668, because the condition on line 667 was never true

668 raise xs_errors.XenError('NoRootDev')

669 return rootdev

670

671

672def getrootdevID():

673 rootdev = getrootdev()

674 try:

675 rootdevID = scsiutil.getSCSIid(rootdev)

676 except:

677 SMlog("util.getrootdevID: Unable to verify serial or SCSIid of device: %s" \

678 % rootdev)

679 return ''

680

681 if not len(rootdevID):

682 SMlog("util.getrootdevID: Unable to identify scsi device [%s] via scsiID" \

683 % rootdev)

684

685 return rootdevID

686

687

688def get_localAPI_session():

689 # First acquire a valid session

690 session = XenAPI.xapi_local()

691 try:

692 session.xenapi.login_with_password('root', '', '', 'SM')

693 except:

694 raise xs_errors.XenError('APISession')

695 return session

696

697

698def get_this_host():

699 uuid = None

700 f = open("/etc/xensource-inventory", 'r')

701 for line in f.readlines():

702 if line.startswith("INSTALLATION_UUID"):

703 uuid = line.split("'")[1]

704 f.close()

705 return uuid

706

707

708def get_master_ref(session):

709 pools = session.xenapi.pool.get_all()

710 return session.xenapi.pool.get_master(pools[0])

711

712

713def is_master(session):

714 return get_this_host_ref(session) == get_master_ref(session)

715

716

717def get_localhost_ref(session):

718 filename = '/etc/xensource-inventory'

719 try:

720 f = open(filename, 'r')

721 except:

722 raise xs_errors.XenError('EIO', \

723 opterr="Unable to open inventory file [%s]" % filename)

724 domid = ''

725 for line in filter(match_domain_id, f.readlines()):

726 domid = line.split("'")[1]

727 if not domid:

728 raise xs_errors.XenError('APILocalhost')

729

730 vms = session.xenapi.VM.get_all_records_where('field "uuid" = "%s"' % domid)

731 for vm in vms:

732 record = vms[vm]

733 if record["uuid"] == domid:

734 hostid = record["resident_on"]

735 return hostid

736 raise xs_errors.XenError('APILocalhost')

737

738

739def match_domain_id(s):

740 regex = re.compile("^CONTROL_DOMAIN_UUID")

741 return regex.search(s, 0)

742

743

744def get_hosts_attached_on(session, vdi_uuids):

745 host_refs = {}

746 for vdi_uuid in vdi_uuids:

747 try:

748 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid)

749 except XenAPI.Failure:

750 SMlog("VDI %s not in db, ignoring" % vdi_uuid)

751 continue

752 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref)

753 for key in [x for x in sm_config.keys() if x.startswith('host_')]:

754 host_refs[key[len('host_'):]] = True

755 return host_refs.keys()

756

757def get_this_host_address(session):

758 host_uuid = get_this_host()

759 host_ref = session.xenapi.host.get_by_uuid(host_uuid)

760 return session.xenapi.host.get_record(host_ref)['address']

761

762def get_host_addresses(session):

763 addresses = []

764 hosts = session.xenapi.host.get_all_records()

765 for record in hosts.values():

766 addresses.append(record['address'])

767 return addresses

768

769def get_this_host_ref(session):

770 host_uuid = get_this_host()

771 host_ref = session.xenapi.host.get_by_uuid(host_uuid)

772 return host_ref

773

774

775def get_slaves_attached_on(session, vdi_uuids):

776 "assume this host is the SR master"

777 host_refs = get_hosts_attached_on(session, vdi_uuids)

778 master_ref = get_this_host_ref(session)

779 return [x for x in host_refs if x != master_ref]

780

781def get_enabled_hosts(session):

782 """

783 Returns a list of host refs that are enabled in the pool.

784 """

785 return list(session.xenapi.host.get_all_records_where('field "enabled" = "true"').keys())

786

787def get_online_hosts(session):

788 online_hosts = []

789 hosts = session.xenapi.host.get_all_records()

790 for host_ref, host_rec in hosts.items():

791 metricsRef = host_rec["metrics"]

792 metrics = session.xenapi.host_metrics.get_record(metricsRef)

793 if metrics["live"]:

794 online_hosts.append(host_ref)

795 return online_hosts

796

797

798def get_all_slaves(session):

799 "assume this host is the SR master"

800 host_refs = get_online_hosts(session)

801 master_ref = get_this_host_ref(session)

802 return [x for x in host_refs if x != master_ref]

803

804

805def is_attached_rw(sm_config):

806 for key, val in sm_config.items():

807 if key.startswith("host_") and val == "RW":

808 return True

809 return False

810

811

812def attached_as(sm_config):

813 for key, val in sm_config.items():

814 if key.startswith("host_") and (val == "RW" or val == "RO"): 814 ↛ 815line 814 didn't jump to line 815, because the condition on line 814 was never true

815 return val

816

817

818def find_my_pbd_record(session, host_ref, sr_ref):

819 try:

820 pbds = session.xenapi.PBD.get_all_records()

821 for pbd_ref in pbds.keys():

822 if pbds[pbd_ref]['host'] == host_ref and pbds[pbd_ref]['SR'] == sr_ref:

823 return [pbd_ref, pbds[pbd_ref]]

824 return None

825 except Exception as e:

826 SMlog("Caught exception while looking up PBD for host %s SR %s: %s" % (str(host_ref), str(sr_ref), str(e)))

827 return None

828

829

830def find_my_pbd(session, host_ref, sr_ref):

831 ret = find_my_pbd_record(session, host_ref, sr_ref)

832 if ret is not None:

833 return ret[0]

834 else:

835 return None

836

837

838def test_hostPBD_devs(session, sr_uuid, devs):

839 host = get_localhost_ref(session)

840 sr = session.xenapi.SR.get_by_uuid(sr_uuid)

841 try:

842 pbds = session.xenapi.PBD.get_all_records()

843 except:

844 raise xs_errors.XenError('APIPBDQuery')

845 for dev in devs.split(','):

846 for pbd in pbds:

847 record = pbds[pbd]

848 # it's ok if it's *our* PBD

849 if record["SR"] == sr:

850 break

851 if record["host"] == host:

852 devconfig = record["device_config"]

853 if 'device' in devconfig:

854 for device in devconfig['device'].split(','):

855 if os.path.realpath(device) == os.path.realpath(dev):

856 return True

857 return False

858

859

860def test_hostPBD_lun(session, targetIQN, LUNid):

861 host = get_localhost_ref(session)

862 try:

863 pbds = session.xenapi.PBD.get_all_records()

864 except:

865 raise xs_errors.XenError('APIPBDQuery')

866 for pbd in pbds:

867 record = pbds[pbd]

868 if record["host"] == host:

869 devconfig = record["device_config"]

870 if 'targetIQN' in devconfig and 'LUNid' in devconfig:

871 if devconfig['targetIQN'] == targetIQN and \

872 devconfig['LUNid'] == LUNid:

873 return True

874 return False

875

876

877def test_SCSIid(session, sr_uuid, SCSIid):

878 if sr_uuid is not None:

879 sr = session.xenapi.SR.get_by_uuid(sr_uuid)

880 try:

881 pbds = session.xenapi.PBD.get_all_records()

882 except:

883 raise xs_errors.XenError('APIPBDQuery')

884 for pbd in pbds:

885 record = pbds[pbd]

886 # it's ok if it's *our* PBD

887 # During FC SR creation, devscan.py passes sr_uuid as None

888 if sr_uuid is not None:

889 if record["SR"] == sr:

890 break

891 devconfig = record["device_config"]

892 sm_config = session.xenapi.SR.get_sm_config(record["SR"])

893 if 'SCSIid' in devconfig and devconfig['SCSIid'] == SCSIid:

894 return True

895 elif 'SCSIid' in sm_config and sm_config['SCSIid'] == SCSIid:

896 return True

897 elif 'scsi-' + SCSIid in sm_config:

898 return True

899 return False

900

901

902class TimeoutException(SMException):

903 pass

904

905

906def timeout_call(timeoutseconds, function, *arguments):

907 def handler(signum, frame):

908 raise TimeoutException()

909 signal.signal(signal.SIGALRM, handler)

910 signal.alarm(timeoutseconds)

911 try:

912 return function(*arguments)

913 finally:

914 signal.alarm(0)

915

916

917def _incr_iscsiSR_refcount(targetIQN, uuid):

918 if not os.path.exists(ISCSI_REFDIR):

919 os.mkdir(ISCSI_REFDIR)

920 filename = os.path.join(ISCSI_REFDIR, targetIQN)

921 try:

922 f = open(filename, 'a+')

923 except:

924 raise xs_errors.XenError('LVMRefCount', \

925 opterr='file %s' % filename)

926

927 f.seek(0)

928 found = False

929 refcount = 0

930 for line in filter(match_uuid, f.readlines()):

931 refcount += 1

932 if line.find(uuid) != -1:

933 found = True

934 if not found:

935 f.write("%s\n" % uuid)

936 refcount += 1

937 f.close()

938 return refcount

939

940

941def _decr_iscsiSR_refcount(targetIQN, uuid):

942 filename = os.path.join(ISCSI_REFDIR, targetIQN)

943 if not os.path.exists(filename):

944 return 0

945 try:

946 f = open(filename, 'a+')

947 except:

948 raise xs_errors.XenError('LVMRefCount', \

949 opterr='file %s' % filename)

950

951 f.seek(0)

952 output = []

953 refcount = 0

954 for line in filter(match_uuid, f.readlines()):

955 if line.find(uuid) == -1:

956 output.append(line.rstrip())

957 refcount += 1

958 if not refcount:

959 os.unlink(filename)

960 return refcount

961

962 # Re-open file and truncate

963 f.close()

964 f = open(filename, 'w')

965 for i in range(0, refcount):

966 f.write("%s\n" % output[i])

967 f.close()

968 return refcount

969

970

971# The agent enforces 1 PBD per SR per host, so we

972# check for active SR entries not attached to this host

973def test_activePoolPBDs(session, host, uuid):

974 try:

975 pbds = session.xenapi.PBD.get_all_records()

976 except:

977 raise xs_errors.XenError('APIPBDQuery')

978 for pbd in pbds:

979 record = pbds[pbd]

980 if record["host"] != host and record["SR"] == uuid \

981 and record["currently_attached"]:

982 return True

983 return False

984

985

986def remove_mpathcount_field(session, host_ref, sr_ref, SCSIid):

987 try:

988 pbdref = find_my_pbd(session, host_ref, sr_ref)

989 if pbdref is not None:

990 key = "mpath-" + SCSIid

991 session.xenapi.PBD.remove_from_other_config(pbdref, key)

992 except:

993 pass

994

995

996def kickpipe_mpathcount():

997 """

998 Issue a kick to the mpathcount service. This will ensure that mpathcount runs

999 shortly to update the multipath config records, if it was not already activated

1000 by a UDEV event.

1001 """

1002 cmd = [CMD_KICKPIPE, "mpathcount"]

1003 (rc, stdout, stderr) = doexec(cmd)

1004 return (rc == 0)

1005

1006

1007def _testHost(hostname, port, errstring):

1008 SMlog("_testHost: Testing host/port: %s,%d" % (hostname, port))

1009 try:

1010 sockinfo = socket.getaddrinfo(hostname, int(port))[0]

1011 except:

1012 logException('Exception occured getting IP for %s' % hostname)

1013 raise xs_errors.XenError('DNSError')

1014

1015 timeout = 5

1016

1017 sock = socket.socket(sockinfo[0], socket.SOCK_STREAM)

1018 # Only allow the connect to block for up to timeout seconds

1019 sock.settimeout(timeout)

1020 try:

1021 sock.connect(sockinfo[4])

1022 # Fix for MS storage server bug

1023 sock.send(b'\n')

1024 sock.close()

1025 except socket.error as reason:

1026 SMlog("_testHost: Connect failed after %d seconds (%s) - %s" \

1027 % (timeout, hostname, reason))

1028 raise xs_errors.XenError(errstring)

1029

1030

1031def match_scsiID(s, id):

1032 regex = re.compile(id)

1033 return regex.search(s, 0)

1034

1035

1036def _isSCSIid(s):

1037 regex = re.compile("^scsi-")

1038 return regex.search(s, 0)

1039

1040

1041def test_scsiserial(session, device):

1042 device = os.path.realpath(device)

1043 if not scsiutil._isSCSIdev(device):

1044 SMlog("util.test_scsiserial: Not a serial device: %s" % device)

1045 return False

1046 serial = ""

1047 try:

1048 serial += scsiutil.getserial(device)

1049 except:

1050 # Error allowed, SCSIid is the important one

1051 pass

1052

1053 try:

1054 scsiID = scsiutil.getSCSIid(device)

1055 except:

1056 SMlog("util.test_scsiserial: Unable to verify serial or SCSIid of device: %s" \

1057 % device)

1058 return False

1059 if not len(scsiID):

1060 SMlog("util.test_scsiserial: Unable to identify scsi device [%s] via scsiID" \

1061 % device)

1062 return False

1063

1064 try:

1065 SRs = session.xenapi.SR.get_all_records()

1066 except:

1067 raise xs_errors.XenError('APIFailure')

1068 for SR in SRs:

1069 record = SRs[SR]

1070 conf = record["sm_config"]

1071 if 'devserial' in conf:

1072 for dev in conf['devserial'].split(','):

1073 if _isSCSIid(dev):

1074 if match_scsiID(dev, scsiID):

1075 return True

1076 elif len(serial) and dev == serial:

1077 return True

1078 return False

1079

1080

1081def default(self, field, thunk):

1082 try:

1083 return getattr(self, field)

1084 except:

1085 return thunk()

1086

1087

1088def list_VDI_records_in_sr(sr):

1089 """Helper function which returns a list of all VDI records for this SR

1090 stored in the XenAPI server, useful for implementing SR.scan"""

1091 sr_ref = sr.session.xenapi.SR.get_by_uuid(sr.uuid)

1092 vdis = sr.session.xenapi.VDI.get_all_records_where("field \"SR\" = \"%s\"" % sr_ref)

1093 return vdis

1094

1095

1096# Given a partition (e.g. sda1), get a disk name:

1097def diskFromPartition(partition):

1098 # check whether this is a device mapper device (e.g. /dev/dm-0)

1099 m = re.match('(/dev/)?(dm-[0-9]+)(p[0-9]+)?$', partition)

1100 if m is not None: 1100 ↛ 1101line 1100 didn't jump to line 1101, because the condition on line 1100 was never true

1101 return m.group(2)

1102

1103 numlen = 0 # number of digit characters

1104 m = re.match(r"\D+(\d+)", partition)

1105 if m is not None: 1105 ↛ 1106line 1105 didn't jump to line 1106, because the condition on line 1105 was never true

1106 numlen = len(m.group(1))

1107

1108 # is it a cciss?

1109 if True in [partition.startswith(x) for x in ['cciss', 'ida', 'rd']]: 1109 ↛ 1110line 1109 didn't jump to line 1110, because the condition on line 1109 was never true

1110 numlen += 1 # need to get rid of trailing 'p'

1111

1112 # is it a mapper path?

1113 if partition.startswith("mapper"): 1113 ↛ 1114line 1113 didn't jump to line 1114, because the condition on line 1113 was never true

1114 if re.search("p[0-9]*$", partition):

1115 numlen = len(re.match(r"\d+", partition[::-1]).group(0)) + 1

1116 SMlog("Found mapper part, len %d" % numlen)

1117 else:

1118 numlen = 0

1119

1120 # is it /dev/disk/by-id/XYZ-part<k>?

1121 if partition.startswith("disk/by-id"): 1121 ↛ 1122line 1121 didn't jump to line 1122, because the condition on line 1121 was never true

1122 return partition[:partition.rfind("-part")]

1123

1124 return partition[:len(partition) - numlen]

1125

1126

1127def dom0_disks():

1128 """Disks carrying dom0, e.g. ['/dev/sda']"""

1129 disks = []

1130 with open("/etc/mtab", 'r') as f:

1131 for line in f:

1132 (dev, mountpoint, fstype, opts, freq, passno) = line.split(' ')

1133 if mountpoint == '/':

1134 disk = diskFromPartition(dev)

1135 if not (disk in disks):

1136 disks.append(disk)

1137 SMlog("Dom0 disks: %s" % disks)

1138 return disks

1139

1140

1141def set_scheduler_sysfs_node(node, scheds):

1142 """

1143 Set the scheduler for a sysfs node (e.g. '/sys/block/sda')

1144 according to prioritized list schedulers

1145 Try to set the first item, then fall back to the next on failure

1146 """

1147

1148 path = os.path.join(node, "queue", "scheduler")

1149 if not os.path.exists(path): 1149 ↛ 1153line 1149 didn't jump to line 1153, because the condition on line 1149 was never false

1150 SMlog("no path %s" % path)

1151 return

1152

1153 stored_error = None

1154 for sched in scheds:

1155 try:

1156 with open(path, 'w') as file:

1157 file.write("%s\n" % sched)

1158 SMlog("Set scheduler to [%s] on [%s]" % (sched, node))

1159 return

1160 except (OSError, IOError) as err:

1161 stored_error = err

1162

1163 SMlog("Error setting schedulers to [%s] on [%s], %s" % (scheds, node, str(stored_error)))

1164

1165

1166def set_scheduler(dev, schedulers=None):

1167 if schedulers is None: 1167 ↛ 1170line 1167 didn't jump to line 1170, because the condition on line 1167 was never false

1168 schedulers = ["none", "noop"]

1169

1170 devices = []

1171 if not scsiutil.match_dm(dev): 1171 ↛ 1175line 1171 didn't jump to line 1175, because the condition on line 1171 was never false

1172 # Remove partition numbers

1173 devices.append(diskFromPartition(dev).replace('/', '!'))

1174 else:

1175 rawdev = diskFromPartition(dev)

1176 devices = [os.path.realpath(x)[5:] for x in scsiutil._genReverseSCSIidmap(rawdev.split('/')[-1])]

1177

1178 for d in devices:

1179 set_scheduler_sysfs_node("/sys/block/%s" % d, schedulers)

1180

1181

1182# This function queries XAPI for the existing VDI records for this SR

1183def _getVDIs(srobj):

1184 VDIs = []

1185 try:

1186 sr_ref = getattr(srobj, 'sr_ref')

1187 except AttributeError:

1188 return VDIs

1189

1190 refs = srobj.session.xenapi.SR.get_VDIs(sr_ref)

1191 for vdi in refs:

1192 ref = srobj.session.xenapi.VDI.get_record(vdi)

1193 ref['vdi_ref'] = vdi

1194 VDIs.append(ref)

1195 return VDIs

1196

1197

1198def _getVDI(srobj, vdi_uuid):

1199 vdi = srobj.session.xenapi.VDI.get_by_uuid(vdi_uuid)

1200 ref = srobj.session.xenapi.VDI.get_record(vdi)

1201 ref['vdi_ref'] = vdi

1202 return ref

1203

1204

1205def _convertDNS(name):

1206 addr = socket.getaddrinfo(name, None)[0][4][0]

1207 return addr

1208

1209

1210def _containsVDIinuse(srobj):

1211 VDIs = _getVDIs(srobj)

1212 for vdi in VDIs:

1213 if not vdi['managed']:

1214 continue

1215 sm_config = vdi['sm_config']

1216 if 'SRRef' in sm_config:

1217 try:

1218 PBDs = srobj.session.xenapi.SR.get_PBDs(sm_config['SRRef'])

1219 for pbd in PBDs:

1220 record = PBDs[pbd]

1221 if record["host"] == srobj.host_ref and \

1222 record["currently_attached"]:

1223 return True

1224 except:

1225 pass

1226 return False

1227

1228

1229def isVDICommand(cmd):

1230 if cmd is None or cmd in ["vdi_attach", "vdi_detach",

1231 "vdi_activate", "vdi_deactivate",

1232 "vdi_epoch_begin", "vdi_epoch_end"]:

1233 return True

1234 else:

1235 return False

1236

1237

1238#########################

1239# Daemon helper functions

1240def p_id_fork():

1241 try:

1242 p_id = os.fork()

1243 except OSError as e:

1244 print("Fork failed: %s (%d)" % (e.strerror, e.errno))

1245 sys.exit(-1)

1246

1247 if (p_id == 0):

1248 os.setsid()

1249 try:

1250 p_id = os.fork()

1251 except OSError as e:

1252 print("Fork failed: %s (%d)" % (e.strerror, e.errno))

1253 sys.exit(-1)

1254 if (p_id == 0):

1255 os.chdir('/opt/xensource/sm')

1256 os.umask(0)

1257 else:

1258 os._exit(0)

1259 else:

1260 os._exit(0)

1261

1262

1263def daemon():

1264 p_id_fork()

1265 # Query the max file descriptor parameter for this process

1266 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]

1267

1268 # Close any fds that are open

1269 for fd in range(0, maxfd):

1270 try:

1271 os.close(fd)

1272 except:

1273 pass

1274

1275 # Redirect STDIN to STDOUT and STDERR

1276 os.open('/dev/null', os.O_RDWR)

1277 os.dup2(0, 1)

1278 os.dup2(0, 2)

1279

1280################################################################################

1281#

1282# Fist points

1283#

1284

1285# * The global variable 'fistpoint' define the list of all possible fistpoints;

1286#

1287# * To activate a fistpoint called 'name', you need to create the file '/tmp/fist_name'

1288# on the SR master;

1289#

1290# * At the moment, activating a fist point can lead to two possible behaviors:

1291# - if '/tmp/fist_LVHDRT_exit' exists, then the function called during the fistpoint is _exit;

1292# - otherwise, the function called is _pause.

1293

1294def _pause(secs, name):

1295 SMlog("Executing fist point %s: sleeping %d seconds ..." % (name, secs))

1296 time.sleep(secs)

1297 SMlog("Executing fist point %s: done" % name)

1298

1299

1300def _exit(name):

1301 SMlog("Executing fist point %s: exiting the current process ..." % name)

1302 raise xs_errors.XenError('FistPoint', opterr='%s' % name)

1303

1304

1305class FistPoint:

1306 def __init__(self, points):

1307 #SMlog("Fist points loaded")

1308 self.points = points

1309

1310 def is_legal(self, name):

1311 return (name in self.points)

1312

1313 def is_active(self, name):

1314 return os.path.exists("/tmp/fist_%s" % name)

1315

1316 def mark_sr(self, name, sruuid, started):

1317 session = get_localAPI_session()

1318 try:

1319 sr = session.xenapi.SR.get_by_uuid(sruuid)

1320

1321 if started:

1322 session.xenapi.SR.add_to_other_config(sr, name, "active")

1323 else:

1324 session.xenapi.SR.remove_from_other_config(sr, name)

1325 finally:

1326 session.xenapi.session.logout()

1327

1328 def activate(self, name, sruuid):

1329 if name in self.points:

1330 if self.is_active(name):

1331 self.mark_sr(name, sruuid, True)

1332 if self.is_active("LVHDRT_exit"): 1332 ↛ 1333line 1332 didn't jump to line 1333, because the condition on line 1332 was never true

1333 self.mark_sr(name, sruuid, False)

1334 _exit(name)

1335 else:

1336 _pause(FIST_PAUSE_PERIOD, name)

1337 self.mark_sr(name, sruuid, False)

1338 else:

1339 SMlog("Unknown fist point: %s" % name)

1340

1341 def activate_custom_fn(self, name, fn):

1342 if name in self.points: 1342 ↛ 1348line 1342 didn't jump to line 1348, because the condition on line 1342 was never false

1343 if self.is_active(name): 1343 ↛ 1344line 1343 didn't jump to line 1344, because the condition on line 1343 was never true

1344 SMlog("Executing fist point %s: starting ..." % name)

1345 fn()

1346 SMlog("Executing fist point %s: done" % name)

1347 else:

1348 SMlog("Unknown fist point: %s" % name)

1349

1350

1351def list_find(f, seq):

1352 for item in seq:

1353 if f(item):

1354 return item

1355

1356GCPAUSE_FISTPOINT = "GCLoop_no_pause"

1357

1358fistpoint = FistPoint(["LVHDRT_finding_a_suitable_pair",

1359 "LVHDRT_inflating_the_parent",

1360 "LVHDRT_resizing_while_vdis_are_paused",

1361 "LVHDRT_coalescing_VHD_data",

1362 "LVHDRT_coalescing_before_inflate_grandparent",

1363 "LVHDRT_relinking_grandchildren",

1364 "LVHDRT_before_create_relink_journal",

1365 "LVHDRT_xapiSM_serialization_tests",

1366 "LVHDRT_clone_vdi_after_create_journal",

1367 "LVHDRT_clone_vdi_after_shrink_parent",

1368 "LVHDRT_clone_vdi_after_first_snap",

1369 "LVHDRT_clone_vdi_after_second_snap",

1370 "LVHDRT_clone_vdi_after_parent_hidden",

1371 "LVHDRT_clone_vdi_after_parent_ro",

1372 "LVHDRT_clone_vdi_before_remove_journal",

1373 "LVHDRT_clone_vdi_after_lvcreate",

1374 "LVHDRT_clone_vdi_before_undo_clone",

1375 "LVHDRT_clone_vdi_after_undo_clone",

1376 "LVHDRT_inflate_after_create_journal",

1377 "LVHDRT_inflate_after_setSize",

1378 "LVHDRT_inflate_after_zeroOut",

1379 "LVHDRT_inflate_after_setSizePhys",

1380 "LVHDRT_inflate_after_setSizePhys",

1381 "LVHDRT_coaleaf_before_coalesce",

1382 "LVHDRT_coaleaf_after_coalesce",

1383 "LVHDRT_coaleaf_one_renamed",

1384 "LVHDRT_coaleaf_both_renamed",

1385 "LVHDRT_coaleaf_after_vdirec",

1386 "LVHDRT_coaleaf_before_delete",

1387 "LVHDRT_coaleaf_after_delete",

1388 "LVHDRT_coaleaf_before_remove_j",

1389 "LVHDRT_coaleaf_undo_after_rename",

1390 "LVHDRT_coaleaf_undo_after_rename2",

1391 "LVHDRT_coaleaf_undo_after_refcount",

1392 "LVHDRT_coaleaf_undo_after_deflate",

1393 "LVHDRT_coaleaf_undo_end",

1394 "LVHDRT_coaleaf_stop_after_recovery",

1395 "LVHDRT_coaleaf_finish_after_inflate",

1396 "LVHDRT_coaleaf_finish_end",

1397 "LVHDRT_coaleaf_delay_1",

1398 "LVHDRT_coaleaf_delay_2",

1399 "LVHDRT_coaleaf_delay_3",

1400 "testsm_clone_allow_raw",

1401 "xenrt_default_vdi_type_legacy",

1402 "blktap_activate_inject_failure",

1403 "blktap_activate_error_handling",

1404 GCPAUSE_FISTPOINT,

1405 "cleanup_coalesceVHD_inject_failure",

1406 "cleanup_tracker_no_progress",

1407 "FileSR_fail_hardlink",

1408 "FileSR_fail_snap1",

1409 "FileSR_fail_snap2",

1410 "LVM_journaler_exists",

1411 "LVM_journaler_none",

1412 "LVM_journaler_badname",

1413 "LVM_journaler_readfail",

1414 "LVM_journaler_writefail"])

1415

1416

1417def set_dirty(session, sr):

1418 try:

1419 session.xenapi.SR.add_to_other_config(sr, "dirty", "")

1420 SMlog("set_dirty %s succeeded" % (repr(sr)))

1421 except:

1422 SMlog("set_dirty %s failed (flag already set?)" % (repr(sr)))

1423

1424

1425def doesFileHaveOpenHandles(fileName):

1426 SMlog("Entering doesFileHaveOpenHandles with file: %s" % fileName)

1427 (retVal, processAndPidTuples) = \

1428 findRunningProcessOrOpenFile(fileName, False)

1429

1430 if not retVal:

1431 SMlog("Failed to determine if file %s has open handles." % \

1432 fileName)

1433 # err on the side of caution

1434 return True

1435 else:

1436 if len(processAndPidTuples) > 0:

1437 return True

1438 else:

1439 return False

1440

1441

1442# extract SR uuid from the passed in devmapper entry and return

1443# /dev/mapper/VG_XenStorage--c3d82e92--cb25--c99b--b83a--482eebab4a93-MGT

1444def extractSRFromDevMapper(path):

1445 try:

1446 path = os.path.basename(path)

1447 path = path[len('VG_XenStorage-') + 1:]

1448 path = path.replace('--', '/')

1449 path = path[0:path.rfind('-')]

1450 return path.replace('/', '-')

1451 except:

1452 return ''

1453

1454

1455def pid_is_alive(pid):

1456 """

1457 Try to kill PID with signal 0.

1458 If we succeed, the PID is alive, so return True.

1459 If we get an EPERM error, the PID is alive but we are not allowed to

1460 signal it. Still return true.

1461 Any other error (e.g. ESRCH), return False

1462 """

1463 try:

1464 os.kill(pid, 0)

1465 return True

1466 except OSError as e:

1467 if e.errno == errno.EPERM:

1468 return True

1469 return False

1470

1471

1472# Looks at /proc and figures either

1473# If a process is still running (default), returns open file names

1474# If any running process has open handles to the given file (process = False)

1475# returns process names and pids

1476def findRunningProcessOrOpenFile(name, process=True):

1477 retVal = True

1478 links = []

1479 processandpids = []

1480 sockets = set()

1481 try:

1482 SMlog("Entering findRunningProcessOrOpenFile with params: %s" % \

1483 [name, process])

1484

1485 # Look at all pids

1486 pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]

1487 for pid in sorted(pids):

1488 try:

1489 try:

1490 f = None

1491 f = open(os.path.join('/proc', pid, 'cmdline'), 'r')

1492 prog = f.read()[:-1]

1493 if prog: 1493 ↛ 1502line 1493 didn't jump to line 1502, because the condition on line 1493 was never false

1494 # Just want the process name

1495 argv = prog.split('\x00')

1496 prog = argv[0]

1497 except IOError as e:

1498 if e.errno in (errno.ENOENT, errno.ESRCH):

1499 SMlog("ERROR %s reading %s, ignore" % (e.errno, pid))

1500 continue

1501 finally:

1502 if f is not None: 1502 ↛ 1487, 1502 ↛ 15052 missed branches: 1) line 1502 didn't jump to line 1487, because the continue on line 1500 wasn't executed, 2) line 1502 didn't jump to line 1505, because the condition on line 1502 was never false

1503 f.close() 1503 ↛ 1487line 1503 didn't jump to line 1487, because the continue on line 1500 wasn't executed

1504

1505 try:

1506 fd_dir = os.path.join('/proc', pid, 'fd')

1507 files = os.listdir(fd_dir)

1508 except OSError as e:

1509 if e.errno in (errno.ENOENT, errno.ESRCH):

1510 SMlog("ERROR %s reading fds for %s, ignore" % (e.errno, pid))

1511 # Ignore pid that are no longer valid

1512 continue

1513 else:

1514 raise

1515

1516 for file in files:

1517 try:

1518 link = os.readlink(os.path.join(fd_dir, file))

1519 except OSError:

1520 continue

1521

1522 if process: 1522 ↛ 1527line 1522 didn't jump to line 1527, because the condition on line 1522 was never false

1523 if name == prog: 1523 ↛ 1516line 1523 didn't jump to line 1516, because the condition on line 1523 was never false

1524 links.append(link)

1525 else:

1526 # need to return process name and pid tuples

1527 if link == name:

1528 processandpids.append((prog, pid))

1529

1530 # Get the connected sockets

1531 if name == prog:

1532 sockets.update(get_connected_sockets(pid))

1533

1534 # We will only have a non-empty processandpids if some fd entries were found.

1535 # Before returning them, verify that all the PIDs in question are properly alive.

1536 # There is no specific guarantee of when a PID's /proc directory will disappear

1537 # when it exits, particularly relative to filedescriptor cleanup, so we want to

1538 # make sure we're not reporting a false positive.

1539 processandpids = [x for x in processandpids if pid_is_alive(int(x[1]))]

1540 for pp in processandpids: 1540 ↛ 1541line 1540 didn't jump to line 1541, because the loop on line 1540 never started

1541 SMlog(f"File {name} has an open handle with process {pp[0]} with pid {pp[1]}")

1542

1543 except Exception as e:

1544 SMlog("Exception checking running process or open file handles. " \

1545 "Error: %s" % str(e))

1546 retVal = False

1547

1548 if process: 1548 ↛ 1551line 1548 didn't jump to line 1551, because the condition on line 1548 was never false

1549 return retVal, links, sockets

1550 else:

1551 return retVal, processandpids

1552

1553

1554def get_connected_sockets(pid):

1555 sockets = set()

1556 try:

1557 # Lines in /proc/<pid>/net/unix are formatted as follows

1558 # (see Linux source net/unix/af_unix.c, unix_seq_show() )

1559 # - Pointer address to socket (hex)

1560 # - Refcount (HEX)

1561 # - 0

1562 # - State (HEX, 0 or __SO_ACCEPTCON)

1563 # - Type (HEX - but only 0001 of interest)

1564 # - Connection state (HEX - but only 03, SS_CONNECTED of interest)

1565 # - Inode number

1566 # - Path (optional)

1567 open_sock_matcher = re.compile(

1568 r'^[0-9a-f]+: [0-9A-Fa-f]+ [0-9A-Fa-f]+ [0-9A-Fa-f]+ 0001 03 \d+ (.*)$')

1569 with open(

1570 os.path.join('/proc', str(pid), 'net', 'unix'), 'r') as f:

1571 lines = f.readlines()

1572 for line in lines:

1573 match = open_sock_matcher.match(line)

1574 if match:

1575 sockets.add(match[1])

1576 except OSError as e:

1577 if e.errno in (errno.ENOENT, errno.ESRCH):

1578 # Ignore pid that are no longer valid

1579 SMlog("ERROR %s reading sockets for %s, ignore" %

1580 (e.errno, pid))

1581 else:

1582 raise

1583 return sockets

1584

1585

1586def retry(f, maxretry=20, period=3, exceptions=[Exception]):

1587 retries = 0

1588 while True:

1589 try:

1590 return f()

1591 except Exception as e:

1592 for exception in exceptions:

1593 if isinstance(e, exception):

1594 SMlog('Got exception: {}. Retry number: {}'.format(

1595 str(e), retries

1596 ))

1597 break

1598 else:

1599 SMlog('Got bad exception: {}. Raising...'.format(e))

1600 raise e

1601

1602 retries += 1

1603 if retries >= maxretry:

1604 break

1605

1606 time.sleep(period)

1607

1608 return f()

1609

1610

1611def getCslDevPath(svid):

1612 basepath = "/dev/disk/by-csldev/"

1613 if svid.startswith("NETAPP_"):

1614 # special attention for NETAPP SVIDs

1615 svid_parts = svid.split("__")

1616 globstr = basepath + "NETAPP__LUN__" + "*" + svid_parts[2] + "*" + svid_parts[-1] + "*"

1617 else:

1618 globstr = basepath + svid + "*"

1619

1620 return globstr

1621

1622

1623# Use device in /dev pointed to by cslg path which consists of svid

1624def get_scsiid_from_svid(md_svid):

1625 cslg_path = getCslDevPath(md_svid)

1626 abs_path = glob.glob(cslg_path)

1627 if abs_path:

1628 real_path = os.path.realpath(abs_path[0])

1629 return scsiutil.getSCSIid(real_path)

1630 else:

1631 return None

1632

1633

1634def get_isl_scsiids(session):

1635 # Get cslg type SRs

1636 SRs = session.xenapi.SR.get_all_records_where('field "type" = "cslg"')

1637

1638 # Iterate through the SR to get the scsi ids

1639 scsi_id_ret = []

1640 for SR in SRs:

1641 sr_rec = SRs[SR]

1642 # Use the md_svid to get the scsi id

1643 scsi_id = get_scsiid_from_svid(sr_rec['sm_config']['md_svid'])

1644 if scsi_id:

1645 scsi_id_ret.append(scsi_id)

1646

1647 # Get the vdis in the SR and do the same procedure

1648 vdi_recs = session.xenapi.VDI.get_all_records_where('field "SR" = "%s"' % SR)

1649 for vdi_rec in vdi_recs:

1650 vdi_rec = vdi_recs[vdi_rec]

1651 scsi_id = get_scsiid_from_svid(vdi_rec['sm_config']['SVID'])

1652 if scsi_id:

1653 scsi_id_ret.append(scsi_id)

1654

1655 return scsi_id_ret

1656

1657

1658class extractXVA:

1659 # streams files as a set of file and checksum, caller should remove

1660 # the files, if not needed. The entire directory (Where the files

1661 # and checksum) will only be deleted as part of class cleanup.

1662 HDR_SIZE = 512

1663 BLOCK_SIZE = 512

1664 SIZE_LEN = 12 - 1 # To remove \0 from tail

1665 SIZE_OFFSET = 124

1666 ZERO_FILLED_REC = 2

1667 NULL_IDEN = '\x00'

1668 DIR_IDEN = '/'

1669 CHECKSUM_IDEN = '.checksum'

1670 OVA_FILE = 'ova.xml'

1671

1672 # Init gunzips the file using a subprocess, and reads stdout later

1673 # as and when needed

1674 def __init__(self, filename):

1675 self.__extract_path = ''

1676 self.__filename = filename

1677 cmd = 'gunzip -cd %s' % filename

1678 try:

1679 self.spawn_p = subprocess.Popen(

1680 cmd, shell=True, \

1681 stdin=subprocess.PIPE, stdout=subprocess.PIPE, \

1682 stderr=subprocess.PIPE, close_fds=True)

1683 except Exception as e:

1684 SMlog("Error: %s. Uncompress failed for %s" % (str(e), filename))

1685 raise Exception(str(e))

1686

1687 # Create dir to extract the files

1688 self.__extract_path = tempfile.mkdtemp()

1689

1690 def __del__(self):

1691 shutil.rmtree(self.__extract_path)

1692

1693 # Class supports Generator expression. 'for f_name, checksum in getTuple()'

1694 # returns filename, checksum content. Returns filename, '' in case

1695 # of checksum file missing. e.g. ova.xml

1696 def getTuple(self):

1697 zerod_record = 0

1698 ret_f_name = ''

1699 ret_base_f_name = ''

1700

1701 try:

1702 # Read tar file as sets of file and checksum.

1703 while True:

1704 # Read the output of spawned process, or output of gunzip

1705 f_hdr = self.spawn_p.stdout.read(self.HDR_SIZE)

1706

1707 # Break out in case of end of file

1708 if f_hdr == '':

1709 if zerod_record == extractXVA.ZERO_FILLED_REC:

1710 break

1711 else:

1712 SMlog('Error. Expects %d zero records', \

1713 extractXVA.ZERO_FILLED_REC)

1714 raise Exception('Unrecognized end of file')

1715

1716 # Watch out for zero records, two zero records

1717 # denote end of file.

1718 if f_hdr == extractXVA.NULL_IDEN * extractXVA.HDR_SIZE:

1719 zerod_record += 1

1720 continue

1721

1722 f_name = f_hdr[:f_hdr.index(extractXVA.NULL_IDEN)]

1723 # File header may be for a folder, if so ignore the header

1724 if not f_name.endswith(extractXVA.DIR_IDEN):

1725 f_size_octal = f_hdr[extractXVA.SIZE_OFFSET: \

1726 extractXVA.SIZE_OFFSET + extractXVA.SIZE_LEN]

1727 f_size = int(f_size_octal, 8)

1728 if f_name.endswith(extractXVA.CHECKSUM_IDEN):

1729 if f_name.rstrip(extractXVA.CHECKSUM_IDEN) == \

1730 ret_base_f_name:

1731 checksum = self.spawn_p.stdout.read(f_size)

1732 yield(ret_f_name, checksum)

1733 else:

1734 # Expects file followed by its checksum

1735 SMlog('Error. Sequence mismatch starting with %s', \

1736 ret_f_name)

1737 raise Exception( \

1738 'Files out of sequence starting with %s', \

1739 ret_f_name)

1740 else:

1741 # In case of ova.xml, read the contents into a file and

1742 # return the file name to the caller. For other files,

1743 # read the contents into a file, it will

1744 # be used when a .checksum file is encountered.

1745 ret_f_name = '%s/%s' % (self.__extract_path, f_name)

1746 ret_base_f_name = f_name

1747

1748 # Check if the folder exists on the target location,

1749 # else create it.

1750 folder_path = ret_f_name[:ret_f_name.rfind('/')]

1751 if not os.path.exists(folder_path):

1752 os.mkdir(folder_path)

1753

1754 # Store the file to the tmp folder, strip the tail \0

1755 f = open(ret_f_name, 'w')

1756 f.write(self.spawn_p.stdout.read(f_size))

1757 f.close()

1758 if f_name == extractXVA.OVA_FILE:

1759 yield(ret_f_name, '')

1760

1761 # Skip zero'd portion of data block

1762 round_off = f_size % extractXVA.BLOCK_SIZE

1763 if round_off != 0:

1764 zeros = self.spawn_p.stdout.read(

1765 extractXVA.BLOCK_SIZE - round_off)

1766 except Exception as e:

1767 SMlog("Error: %s. File set extraction failed %s" % (str(e), \

1768 self.__filename))

1769

1770 # Kill and Drain stdout of the gunzip process,

1771 # else gunzip might block on stdout

1772 os.kill(self.spawn_p.pid, signal.SIGTERM)

1773 self.spawn_p.communicate()

1774 raise Exception(str(e))

1775

1776illegal_xml_chars = [(0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F),

1777 (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF),

1778 (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF),

1779 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF),

1780 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF),

1781 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF),

1782 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF),

1783 (0x10FFFE, 0x10FFFF)]

1784

1785illegal_ranges = ["%s-%s" % (chr(low), chr(high))

1786 for (low, high) in illegal_xml_chars

1787 if low < sys.maxunicode]

1788

1789illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges))

1790

1791

1792def isLegalXMLString(s):

1793 """Tells whether this is a valid XML string (i.e. it does not contain

1794 illegal XML characters specified in

1795 http://www.w3.org/TR/2004/REC-xml-20040204/#charsets).

1796 """

1797

1798 if len(s) > 0:

1799 return re.search(illegal_xml_re, s) is None

1800 else:

1801 return True

1802

1803

1804def unictrunc(string, max_bytes):

1805 """

1806 Given a string, returns the largest number of elements for a prefix

1807 substring of it, such that the UTF-8 encoding of this substring takes no

1808 more than the given number of bytes.

1809

1810 The string may be given as a unicode string or a UTF-8 encoded byte

1811 string, and the number returned will be in characters or bytes

1812 accordingly. Note that in the latter case, the substring will still be a

1813 valid UTF-8 encoded string (which is to say, it won't have been truncated

1814 part way through a multibyte sequence for a unicode character).

1815

1816 string: the string to truncate

1817 max_bytes: the maximum number of bytes the truncated string can be

1818 """

1819 if isinstance(string, str):

1820 return_chars = True

1821 else:

1822 return_chars = False

1823 string = string.decode('UTF-8')

1824

1825 cur_chars = 0

1826 cur_bytes = 0

1827 for char in string:

1828 charsize = len(char.encode('UTF-8'))

1829 if cur_bytes + charsize > max_bytes:

1830 break

1831 else:

1832 cur_chars += 1

1833 cur_bytes += charsize

1834 return cur_chars if return_chars else cur_bytes

1835

1836

1837def hideValuesInPropMap(propmap, propnames):

1838 """

1839 Worker function: input simple map of prop name/value pairs, and

1840 a list of specific propnames whose values we want to hide.

1841 Loop through the "hide" list, and if any are found, hide the

1842 value and return the altered map.

1843 If none found, return the original map

1844 """

1845 matches = []

1846 for propname in propnames:

1847 if propname in propmap: 1847 ↛ 1848line 1847 didn't jump to line 1848, because the condition on line 1847 was never true

1848 matches.append(propname)

1849

1850 if matches: 1850 ↛ 1851line 1850 didn't jump to line 1851, because the condition on line 1850 was never true

1851 deepCopyRec = copy.deepcopy(propmap)

1852 for match in matches:

1853 deepCopyRec[match] = '******'

1854 return deepCopyRec

1855

1856 return propmap

1857# define the list of propnames whose value we want to hide

1858

1859PASSWD_PROP_KEYS = ['password', 'cifspassword', 'chappassword', 'incoming_chappassword']

1860DEFAULT_SEGMENT_LEN = 950

1861

1862

1863def hidePasswdInConfig(config):

1864 """

1865 Function to hide passwd values in a simple prop map,

1866 for example "device_config"

1867 """

1868 return hideValuesInPropMap(config, PASSWD_PROP_KEYS)

1869

1870

1871def hidePasswdInParams(params, configProp):

1872 """

1873 Function to hide password values in a specified property which

1874 is a simple map of prop name/values, and is itself an prop entry

1875 in a larger property map.

1876 For example, param maps containing "device_config", or

1877 "sm_config", etc

1878 """

1879 params[configProp] = hideValuesInPropMap(params[configProp], PASSWD_PROP_KEYS)

1880 return params

1881

1882

1883def hideMemberValuesInXmlParams(xmlParams, propnames=PASSWD_PROP_KEYS):

1884 """

1885 Function to hide password values in XML params, specifically

1886 for the XML format of incoming params to SR modules.

1887 Uses text parsing: loop through the list of specific propnames

1888 whose values we want to hide, and:

1889 - Assemble a full "prefix" containing each property name, e.g.,

1890 "<member><name>password</name><value>"

1891 - Test the XML if it contains that string, save the index.

1892 - If found, get the index of the ending tag

1893 - Truncate the return string starting with the password value.

1894 - Append the substitute "*******" value string.

1895 - Restore the rest of the original string starting with the end tag.

1896 """

1897 findStrPrefixHead = "<member><name>"

1898 findStrPrefixTail = "</name><value>"

1899 findStrSuffix = "</value>"

1900 strlen = len(xmlParams)

1901

1902 for propname in propnames:

1903 findStrPrefix = findStrPrefixHead + propname + findStrPrefixTail

1904 idx = xmlParams.find(findStrPrefix)

1905 if idx != -1: # if found any of them

1906 idx += len(findStrPrefix)

1907 idx2 = xmlParams.find(findStrSuffix, idx)

1908 if idx2 != -1:

1909 retStr = xmlParams[0:idx]

1910 retStr += "******"

1911 retStr += xmlParams[idx2:strlen]

1912 return retStr

1913 else:

1914 return xmlParams

1915 return xmlParams

1916

1917

1918def splitXmlText(xmlData, segmentLen=DEFAULT_SEGMENT_LEN, showContd=False):

1919 """

1920 Split xml string data into substrings small enough for the

1921 syslog line length limit. Split at tag end markers ( ">" ).

1922 Usage:

1923 strList = []

1924 strList = splitXmlText( longXmlText, maxLineLen ) # maxLineLen is optional

1925 """

1926 remainingData = str(xmlData)

1927

1928 # "Un-pretty-print"

1929 remainingData = remainingData.replace('\n', '')

1930 remainingData = remainingData.replace('\t', '')

1931

1932 remainingChars = len(remainingData)

1933 returnData = ''

1934

1935 thisLineNum = 0

1936 while remainingChars > segmentLen:

1937 thisLineNum = thisLineNum + 1

1938 index = segmentLen

1939 tmpStr = remainingData[:segmentLen]

1940 tmpIndex = tmpStr.rfind('>')

1941 if tmpIndex != -1:

1942 index = tmpIndex + 1

1943

1944 tmpStr = tmpStr[:index]

1945 remainingData = remainingData[index:]

1946 remainingChars = len(remainingData)

1947

1948 if showContd:

1949 if thisLineNum != 1:

1950 tmpStr = '(Cont\'d): ' + tmpStr

1951 tmpStr = tmpStr + ' (Cont\'d):'

1952

1953 returnData += tmpStr + '\n'

1954

1955 if showContd and thisLineNum > 0:

1956 remainingData = '(Cont\'d): ' + remainingData

1957 returnData += remainingData

1958

1959 return returnData

1960

1961

1962def inject_failure():

1963 raise Exception('injected failure')

1964

1965

1966def open_atomic(path, mode=None):

1967 """Atomically creates a file if, and only if it does not already exist.

1968 Leaves the file open and returns the file object.

1969

1970 path: the path to atomically open

1971 mode: "r" (read), "w" (write), or "rw" (read/write)

1972 returns: an open file object"""

1973

1974 assert path

1975

1976 flags = os.O_CREAT | os.O_EXCL

1977 modes = {'r': os.O_RDONLY, 'w': os.O_WRONLY, 'rw': os.O_RDWR}

1978 if mode:

1979 if mode not in modes:

1980 raise Exception('invalid access mode ' + mode)

1981 flags |= modes[mode]

1982 fd = os.open(path, flags)

1983 try:

1984 if mode:

1985 return os.fdopen(fd, mode)

1986 else:

1987 return os.fdopen(fd)

1988 except:

1989 os.close(fd)

1990 raise

1991

1992

1993def isInvalidVDI(exception):

1994 return exception.details[0] == "HANDLE_INVALID" or \

1995 exception.details[0] == "UUID_INVALID"

1996

1997

1998def get_pool_restrictions(session):

1999 """Returns pool restrictions as a map, @session must be already

2000 established."""

2001 return list(session.xenapi.pool.get_all_records().values())[0]['restrictions']

2002

2003

2004def read_caching_is_restricted(session):

2005 """Tells whether read caching is restricted."""

2006 if session is None: 2006 ↛ 2007line 2006 didn't jump to line 2007, because the condition on line 2006 was never true

2007 return True

2008 restrictions = get_pool_restrictions(session)

2009 if 'restrict_read_caching' in restrictions and \ 2009 ↛ 2011line 2009 didn't jump to line 2011, because the condition on line 2009 was never true

2010 restrictions['restrict_read_caching'] == "true":

2011 return True

2012 return False

2013

2014

2015def sessions_less_than_targets(other_config, device_config):

2016 if 'multihomelist' in device_config and 'iscsi_sessions' in other_config:

2017 sessions = int(other_config['iscsi_sessions'])

2018 targets = len(device_config['multihomelist'].split(','))