Task improvements, run fsck in background
[openblackhole/openblackhole-enigma2.git] / lib / python / Components / Harddisk.py
1 from os import system, listdir, statvfs, popen, makedirs, stat, major, minor, path, access
2 from Tools.Directories import SCOPE_HDD, resolveFilename, pathExists
3 from Tools.CList import CList
4 from SystemInfo import SystemInfo
5 import time
6 from Components.Console import Console
7 import Task
8
9 def readFile(filename):
10         file = open(filename)
11         data = file.read().strip()
12         file.close()
13         return data
14
15 def getProcMounts():
16         try:
17                 mounts = open("/proc/mounts")
18         except IOError, ex:
19                 print "[Harddisk] Failed to open /proc/mounts", ex 
20                 return []
21         return [line.strip().split(' ') for line in mounts]
22
23 class Harddisk:
24         DEVTYPE_UDEV = 0
25         DEVTYPE_DEVFS = 1
26
27         def __init__(self, device):
28                 self.device = device
29
30                 if access("/dev/.udev", 0):
31                         self.type = self.DEVTYPE_UDEV
32                 elif access("/dev/.devfsd", 0):
33                         self.type = self.DEVTYPE_DEVFS
34                 else:
35                         print "Unable to determine structure of /dev"
36
37                 self.max_idle_time = 0
38                 self.idle_running = False
39                 self.timer = None
40
41                 self.dev_path = ''
42                 self.disk_path = ''
43                 self.phys_path = path.realpath(self.sysfsPath('device'))
44
45                 if self.type == self.DEVTYPE_UDEV:
46                         self.dev_path = '/dev/' + self.device
47                         self.disk_path = self.dev_path
48
49                 elif self.type == self.DEVTYPE_DEVFS:
50                         tmp = readFile(self.sysfsPath('dev')).split(':')
51                         s_major = int(tmp[0])
52                         s_minor = int(tmp[1])
53                         for disc in listdir("/dev/discs"):
54                                 dev_path = path.realpath('/dev/discs/' + disc)
55                                 disk_path = dev_path + '/disc'
56                                 try:
57                                         rdev = stat(disk_path).st_rdev
58                                 except OSError:
59                                         continue
60                                 if s_major == major(rdev) and s_minor == minor(rdev):
61                                         self.dev_path = dev_path
62                                         self.disk_path = disk_path
63                                         break
64
65                 print "new Harddisk", self.device, '->', self.dev_path, '->', self.disk_path
66                 self.startIdle()
67
68         def __lt__(self, ob):
69                 return self.device < ob.device
70
71         def partitionPath(self, n):
72                 if self.type == self.DEVTYPE_UDEV:
73                         return self.dev_path + n
74                 elif self.type == self.DEVTYPE_DEVFS:
75                         return self.dev_path + '/part' + n
76
77         def sysfsPath(self, filename):
78                 return path.realpath('/sys/block/' + self.device + '/' + filename)
79
80         def stop(self):
81                 if self.timer:
82                         self.timer.stop()
83                         self.timer.callback.remove(self.runIdle)
84
85         def bus(self):
86                 # CF (7025 specific)
87                 if self.type == self.DEVTYPE_UDEV:
88                         ide_cf = False  # FIXME
89                 elif self.type == self.DEVTYPE_DEVFS:
90                         ide_cf = self.device[:2] == "hd" and "host0" not in self.dev_path
91
92                 internal = "pci" in self.phys_path
93
94                 if ide_cf:
95                         ret = "External (CF)"
96                 elif internal:
97                         ret = "Internal"
98                 else:
99                         ret = "External"
100                 return ret
101
102         def diskSize(self):
103                 line = readFile(self.sysfsPath('size'))
104                 try:
105                         cap = int(line)
106                 except:
107                         return 0;
108                 return cap / 1000 * 512 / 1000
109
110         def capacity(self):
111                 cap = self.diskSize()
112                 if cap == 0:
113                         return ""
114                 return "%d.%03d GB" % (cap/1000, cap%1000)
115
116         def model(self):
117                 if self.device[:2] == "hd":
118                         return readFile('/proc/ide/' + self.device + '/model')
119                 elif self.device[:2] == "sd":
120                         vendor = readFile(self.sysfsPath('device/vendor'))
121                         model = readFile(self.sysfsPath('device/model'))
122                         return vendor + '(' + model + ')'
123                 else:
124                         assert False, "no hdX or sdX"
125
126         def free(self):
127                 for parts in getProcMounts():
128                         if path.realpath(parts[0]).startswith(self.dev_path):
129                                 try:
130                                         stat = statvfs(parts[1])
131                                 except OSError:
132                                         continue
133                                 return stat.f_bfree/1000 * stat.f_bsize/1000
134                 return -1
135
136         def numPartitions(self):
137                 numPart = -1
138                 if self.type == self.DEVTYPE_UDEV:
139                         try:
140                                 devdir = listdir('/dev')
141                         except OSError:
142                                 return -1
143                         for filename in devdir:
144                                 if filename.startswith(self.device):
145                                         numPart += 1
146
147                 elif self.type == self.DEVTYPE_DEVFS:
148                         try:
149                                 idedir = listdir(self.dev_path)
150                         except OSError:
151                                 return -1
152                         for filename in idedir:
153                                 if filename.startswith("disc"):
154                                         numPart += 1
155                                 if filename.startswith("part"):
156                                         numPart += 1
157                 return numPart
158
159         def _unmount(self):
160                 self.unmount()
161                 
162         def unmount(self):
163                 cmd = "umount"
164                 for parts in getProcMounts():
165                         if path.realpath(parts[0]).startswith(self.dev_path):
166                                 cmd = ' ' . join([cmd, parts[1]])
167                                 break
168                 else:
169                         # not mounted, return OK
170                         return 0
171                 print "[Harddisk]", cmd
172                 res = system(cmd)
173                 return (res >> 8)
174
175         def createPartition(self):
176                 cmd = 'printf "0,\n;\n;\n;\ny\n" | sfdisk -f ' + self.disk_path
177                 res = system(cmd)
178                 return (res >> 8)
179
180         def mkfs(self):
181                 cmd = "mkfs.ext3 "
182                 size = self.diskSize()
183                 if size > 16 * 1024:
184                         cmd += "-T largefile -O sparse_super "
185                 elif size > 2 * 1024:
186                         cmd += "-T largefile -N %d " % (size * 32)
187                 cmd += "-m0 -O dir_index " + self.partitionPath("1")
188                 res = system(cmd)
189                 return (res >> 8)
190
191         def mount(self):
192                 try:
193                         fstab = open("/etc/fstab")
194                 except IOError:
195                         return -1
196
197                 lines = fstab.readlines()
198                 fstab.close()
199
200                 res = -1
201                 for line in lines:
202                         parts = line.strip().split(" ")
203                         if path.realpath(parts[0]) == self.partitionPath("1"):
204                                 cmd = "mount -t ext3 " + parts[0]
205                                 res = system(cmd)
206                                 return (res >> 8)
207
208                 # device is not in fstab
209                 if self.type == self.DEVTYPE_UDEV:
210                         # we can let udev do the job, re-read the partition table
211                         res = system('/sbin/sfdisk -R ' + self.disk_path)
212                         # give udev some time to make the mount, which it will do asynchronously
213                         from time import sleep
214                         sleep(3)
215
216                 return (res >> 8)
217
218         def createMovieFolder(self):
219                 if not pathExists(resolveFilename(SCOPE_HDD)):
220                         try:
221                                 makedirs(resolveFilename(SCOPE_HDD))
222                         except OSError:
223                                 return -1
224                 return 0
225
226         def fsck(self):
227                 # We autocorrect any failures
228                 # TODO: we could check if the fs is actually ext3
229                 cmd = "fsck.ext3 -f -p " + self.partitionPath("1")
230                 res = system(cmd)
231                 return (res >> 8)
232
233         def killPartition(self, n):
234                 part = self.partitionPath(n)
235                 try:
236                         h = open(part, 'wb')
237                         zero = 512 * '\0'
238                         for i in range(3):
239                                 h.write(zero)
240                         h.close()
241                 except Exception, e:
242                         print "[Harddisk] Failed to write to", part, "error:", e
243
244         errorList = [ _("Everything is fine"), _("Creating partition failed"), _("Mkfs failed"), _("Mount failed"), _("Create movie folder failed"), _("Fsck failed"), _("Please Reboot"), _("Filesystem contains uncorrectable errors"), _("Unmount failed")]
245
246         def createInitializeJob(self):
247                 job = Task.Job(_("Initializing storage device..."))
248
249                 task = Task.PythonTask(job, _("Unmount"))
250                 task.work = self.unmount
251
252                 task = Task.PythonTask(job, _("Kill partition"))
253                 task.work = lambda: self.killPartition("1")
254
255                 task = Task.Task(job, _("Create Partition"))
256                 task.setTool('sfdisk')
257                 task.args.append('-f')
258                 task.args.append(self.disk_path)
259                 task.initial_input = "0,\n;\n;\n;\ny\n"
260                 
261                 task = Task.Task(job, _("Create Filesystem"))
262                 task.setTool("mkfs.ext3")
263                 size = self.diskSize()
264                 if size > 16 * 1024:
265                         task.args += ["-T", "largefile", "-O", "sparse_super"]
266                 elif size > 2 * 1024:
267                         task.args += ["-T", "largefile", "-N", str(size * 32)]
268                 task.args += ["-m0", "-O", "dir_index", self.partitionPath("1")]
269
270                 task = Task.PythonTask(job, _("Mount"))
271                 task.work = self.mount
272
273                 return job
274
275         def initialize(self):
276                 self.unmount()
277
278                 # Udev tries to mount the partition immediately if there is an
279                 # old filesystem on it when fdisk reloads the partition table.
280                 # To prevent that, we overwrite the first 3 sectors of the
281                 # partition, if the partition existed before. That's enough for
282                 # ext3 at least.
283                 self.killPartition("1")
284
285                 if self.createPartition() != 0:
286                         return -1
287
288                 if self.mkfs() != 0:
289                         return -2
290
291                 if self.mount() != 0:
292                         return -3
293
294                 if self.createMovieFolder() != 0:
295                         return -4
296
297                 return 0
298
299         def check(self):
300                 self.unmount()
301
302                 res = self.fsck()
303                 if res & 2 == 2:
304                         return -6
305
306                 if res & 4 == 4:
307                         return -7
308
309                 if res != 0 and res != 1:
310                         # A sum containing 1 will also include a failure
311                         return -5
312
313                 if self.mount() != 0:
314                         return -3
315
316                 return 0
317                 
318         def createCheckJob(self):
319                 job = Task.Job(_("Check storage device..."))
320
321                 task = Task.PythonTask(job, _("Unmount"))
322                 task.work = self.unmount
323
324                 task = Task.Task(job, "fsck")
325                 task.setTool('fsck.ext3')
326                 task.args.append('-f')
327                 task.args.append('-p')
328                 task.args.append(self.partitionPath("1"))
329
330                 task = Task.PythonTask(job, _("Mount"))
331                 task.work = self.mount
332
333                 return job
334
335         def getDeviceDir(self):
336                 return self.dev_path
337
338         def getDeviceName(self):
339                 return self.disk_path
340
341         # the HDD idle poll daemon.
342         # as some harddrives have a buggy standby timer, we are doing this by hand here.
343         # first, we disable the hardware timer. then, we check every now and then if
344         # any access has been made to the disc. If there has been no access over a specifed time,
345         # we set the hdd into standby.
346         def readStats(self):
347                 try:
348                         l = open("/sys/block/%s/stat" % self.device).read()
349                 except IOError:
350                         return -1,-1
351                 (nr_read, _, _, _, nr_write) = l.split()[:5]
352                 return int(nr_read), int(nr_write)
353
354         def startIdle(self):
355                 self.last_access = time.time()
356                 self.last_stat = 0
357                 self.is_sleeping = False
358                 from enigma import eTimer
359
360                 # disable HDD standby timer
361                 Console().ePopen(("hdparm", "hdparm", "-S0", self.disk_path))
362                 self.timer = eTimer()
363                 self.timer.callback.append(self.runIdle)
364                 self.idle_running = True
365                 self.setIdleTime(self.max_idle_time) # kick the idle polling loop
366
367         def runIdle(self):
368                 if not self.max_idle_time:
369                         return
370                 t = time.time()
371
372                 idle_time = t - self.last_access
373
374                 stats = self.readStats()
375                 l = sum(stats)
376
377                 if l != self.last_stat and l >= 0: # access
378                         self.last_stat = l
379                         self.last_access = t
380                         idle_time = 0
381                         self.is_sleeping = False
382
383                 if idle_time >= self.max_idle_time and not self.is_sleeping:
384                         self.setSleep()
385                         self.is_sleeping = True
386
387         def setSleep(self):
388                 Console().ePopen(("hdparm", "hdparm", "-y", self.disk_path))
389
390         def setIdleTime(self, idle):
391                 self.max_idle_time = idle
392                 if self.idle_running:
393                         if not idle:
394                                 self.timer.stop()
395                         else:
396                                 self.timer.start(idle * 100, False)  # poll 10 times per period.
397
398         def isSleeping(self):
399                 return self.is_sleeping
400
401 class Partition:
402         def __init__(self, mountpoint, device = None, description = "", force_mounted = False):
403                 self.mountpoint = mountpoint
404                 self.description = description
405                 self.force_mounted = force_mounted
406                 self.is_hotplug = force_mounted # so far; this might change.
407                 self.device = device
408
409         def stat(self):
410                 return statvfs(self.mountpoint)
411
412         def free(self):
413                 try:
414                         s = self.stat()
415                         return s.f_bavail * s.f_bsize
416                 except OSError:
417                         return None
418
419         def total(self):
420                 try:
421                         s = self.stat()
422                         return s.f_blocks * s.f_bsize
423                 except OSError:
424                         return None
425
426         def mounted(self, mounts = None):
427                 # THANK YOU PYTHON FOR STRIPPING AWAY f_fsid.
428                 # TODO: can os.path.ismount be used?
429                 if self.force_mounted:
430                         return True
431                 if mounts is None:
432                         mounts = getProcMounts()
433                 for parts in mounts:
434                         if parts[1] == self.mountpoint:
435                                 return True
436                 return False
437
438         def filesystem(self):
439                 for fields in getProcMounts():
440                         if fields[1] == self.mountpoint:
441                                 return fields[2]
442                 return ''
443
444 DEVICEDB =  \
445         {"dm8000":
446                 {
447                         # dm8000:
448                         "/devices/platform/brcm-ehci.0/usb1/1-1/1-1.1/1-1.1:1.0": "Front USB Slot",
449                         "/devices/platform/brcm-ehci.0/usb1/1-1/1-1.2/1-1.2:1.0": "Back, upper USB Slot",
450                         "/devices/platform/brcm-ehci.0/usb1/1-1/1-1.3/1-1.3:1.0": "Back, lower USB Slot",
451                         "/devices/platform/brcm-ehci-1.1/usb2/2-1/2-1:1.0/host1/target1:0:0/1:0:0:0": "DVD Drive",
452                 },
453         "dm800":
454         {
455                 # dm800:
456                 "/devices/platform/brcm-ehci.0/usb1/1-2/1-2:1.0": "Upper USB Slot",
457                 "/devices/platform/brcm-ehci.0/usb1/1-1/1-1:1.0": "Lower USB Slot",
458         },
459         "dm7025":
460         {
461                 # dm7025:
462                 "/devices/pci0000:00/0000:00:14.1/ide1/1.0": "CF Card Slot", #hdc
463                 "/devices/pci0000:00/0000:00:14.1/ide0/0.0": "Internal Harddisk"
464         }
465         }
466
467 class HarddiskManager:
468         def __init__(self):
469                 self.hdd = [ ]
470                 self.cd = ""
471                 self.partitions = [ ]
472
473                 self.on_partition_list_change = CList()
474
475                 self.enumerateBlockDevices()
476
477                 # currently, this is just an enumeration of what's possible,
478                 # this probably has to be changed to support automount stuff.
479                 # still, if stuff is mounted into the correct mountpoints by
480                 # external tools, everything is fine (until somebody inserts
481                 # a second usb stick.)
482                 p = [
483                                         ("/media/hdd", _("Harddisk")),
484                                         ("/media/card", _("Card")),
485                                         ("/media/cf", _("Compact Flash")),
486                                         ("/media/mmc1", _("MMC Card")),
487                                         ("/media/net", _("Network Mount")),
488                                         ("/media/net1", _("Network Mount") + " 1"),
489                                         ("/media/net2", _("Network Mount") + " 2"),
490                                         ("/media/net3", _("Network Mount") + " 3"),
491                                         ("/media/ram", _("Ram Disk")),
492                                         ("/media/usb", _("USB Stick")),
493                                         ("/", _("Internal Flash"))
494                                 ]
495
496                 self.partitions.extend([ Partition(mountpoint = x[0], description = x[1]) for x in p ])
497
498         def getBlockDevInfo(self, blockdev):
499                 devpath = "/sys/block/" + blockdev
500                 error = False
501                 removable = False
502                 blacklisted = False
503                 is_cdrom = False
504                 partitions = []
505                 try:
506                         removable = bool(int(readFile(devpath + "/removable")))
507                         dev = int(readFile(devpath + "/dev").split(':')[0])
508                         if dev in (7, 31): # loop, mtdblock
509                                 blacklisted = True
510                         if blockdev[0:2] == 'sr':
511                                 is_cdrom = True
512                         if blockdev[0:2] == 'hd':
513                                 try:
514                                         media = readFile("/proc/ide/%s/media" % blockdev)
515                                         if "cdrom" in media:
516                                                 is_cdrom = True
517                                 except IOError:
518                                         error = True
519                         # check for partitions
520                         if not is_cdrom:
521                                 for partition in listdir(devpath):
522                                         if partition[0:len(blockdev)] != blockdev:
523                                                 continue
524                                         partitions.append(partition)
525                         else:
526                                 self.cd = blockdev
527                 except IOError:
528                         error = True
529                 # check for medium
530                 medium_found = True
531                 try:
532                         open("/dev/" + blockdev).close()
533                 except IOError, err:
534                         if err.errno == 159: # no medium present
535                                 medium_found = False
536
537                 return error, blacklisted, removable, is_cdrom, partitions, medium_found
538
539         def enumerateBlockDevices(self):
540                 print "enumerating block devices..."
541                 for blockdev in listdir("/sys/block"):
542                         error, blacklisted, removable, is_cdrom, partitions, medium_found = self.getBlockDevInfo(blockdev)
543                         print "found block device '%s':" % blockdev, 
544                         if error:
545                                 print "error querying properties"
546                         elif blacklisted:
547                                 print "blacklisted"
548                         elif not medium_found:
549                                 print "no medium"
550                         else:
551                                 print "ok, removable=%s, cdrom=%s, partitions=%s, device=%s" % (removable, is_cdrom, partitions, blockdev)
552
553                                 self.addHotplugPartition(blockdev)
554                                 for part in partitions:
555                                         self.addHotplugPartition(part)
556
557         def getAutofsMountpoint(self, device):
558                 return "/autofs/%s/" % (device)
559
560         def addHotplugPartition(self, device, physdev = None):
561                 if not physdev:
562                         dev, part = self.splitDeviceName(device)
563                         try:
564                                 physdev = path.realpath('/sys/block/' + dev + '/device')[4:]
565                         except OSError:
566                                 physdev = dev
567                                 print "couldn't determine blockdev physdev for device", device
568
569                 # device is the device name, without /dev
570                 # physdev is the physical device path, which we (might) use to determine the userfriendly name
571                 description = self.getUserfriendlyDeviceName(device, physdev)
572
573                 p = Partition(mountpoint = self.getAutofsMountpoint(device), description = description, force_mounted = True, device = device)
574                 self.partitions.append(p)
575                 self.on_partition_list_change("add", p)
576
577                 # see if this is a harddrive
578                 l = len(device)
579                 if l and not device[l-1].isdigit():
580                         error, blacklisted, removable, is_cdrom, partitions, medium_found = self.getBlockDevInfo(device)
581                         if not blacklisted and not is_cdrom and medium_found:
582                                 self.hdd.append(Harddisk(device))
583                                 self.hdd.sort()
584                                 SystemInfo["Harddisk"] = len(self.hdd) > 0
585
586         def removeHotplugPartition(self, device):
587                 mountpoint = self.getAutofsMountpoint(device)
588                 for x in self.partitions[:]:
589                         if x.mountpoint == mountpoint:
590                                 self.partitions.remove(x)
591                                 self.on_partition_list_change("remove", x)
592                 l = len(device)
593                 if l and not device[l-1].isdigit():
594                         for hdd in self.hdd:
595                                 if hdd.device == device:
596                                         hdd.stop()
597                                         self.hdd.remove(hdd)
598                                         break
599                         SystemInfo["Harddisk"] = len(self.hdd) > 0
600
601         def HDDCount(self):
602                 return len(self.hdd)
603
604         def HDDList(self):
605                 list = [ ]
606                 for hd in self.hdd:
607                         hdd = hd.model() + " - " + hd.bus()
608                         cap = hd.capacity()
609                         if cap != "":
610                                 hdd += " (" + cap + ")"
611                         list.append((hdd, hd))
612                 return list
613
614         def getCD(self):
615                 return self.cd
616
617         def getMountedPartitions(self, onlyhotplug = False):
618                 mounts = getProcMounts()
619                 parts = [x for x in self.partitions if (x.is_hotplug or not onlyhotplug) and x.mounted(mounts)]
620                 devs = set([x.device for x in parts])
621                 for devname in devs.copy():
622                         if not devname:
623                                 continue
624                         dev, part = self.splitDeviceName(devname)
625                         if part and dev in devs: # if this is a partition and we still have the wholedisk, remove wholedisk
626                                 devs.remove(dev)
627
628                 # return all devices which are not removed due to being a wholedisk when a partition exists
629                 return [x for x in parts if not x.device or x.device in devs]
630
631         def splitDeviceName(self, devname):
632                 # this works for: sdaX, hdaX, sr0 (which is in fact dev="sr0", part=""). It doesn't work for other names like mtdblock3, but they are blacklisted anyway.
633                 dev = devname[:3]
634                 part = devname[3:]
635                 for p in part:
636                         if not p.isdigit():
637                                 return devname, 0
638                 return dev, part and int(part) or 0
639
640         def getUserfriendlyDeviceName(self, dev, phys):
641                 dev, part = self.splitDeviceName(dev)
642                 description = "External Storage %s" % dev
643                 try:
644                         description = readFile("/sys" + phys + "/model")
645                 except IOError, s:
646                         print "couldn't read model: ", s
647                 from Tools.HardwareInfo import HardwareInfo
648                 for physdevprefix, pdescription in DEVICEDB.get(HardwareInfo().device_name,{}).items():
649                         if phys.startswith(physdevprefix):
650                                 description = pdescription
651
652                 # not wholedisk and not partition 1
653                 if part and part != 1:
654                         description += " (Partition %d)" % part
655                 return description
656
657         def addMountedPartition(self, device, desc):
658                 already_mounted = False
659                 for x in self.partitions[:]:
660                         if x.mountpoint == device:
661                                 already_mounted = True
662                 if not already_mounted:
663                         self.partitions.append(Partition(mountpoint = device, description = desc))
664
665         def removeMountedPartition(self, mountpoint):
666                 for x in self.partitions[:]:
667                         if x.mountpoint == mountpoint:
668                                 self.partitions.remove(x)
669                                 self.on_partition_list_change("remove", x)
670
671 harddiskmanager = HarddiskManager()