Merge branch 'release/0.10.5'

-

README.md

@@ -44,14 +44,69 @@ The _allow remote_ option is to allow remote add and stream of torrents.
 
 ## Todo
 
-* [x] Add RAR streaming support
 * [ ] Better feedback in interface about streams
 * [ ] Better feedback when using API
-* [x] Reverse proxy improvement (e.g. port different than bind port)
+* [ ] Fix problems when removing torrent from Deluge (sea of errors)
+
+# HTTP API Usage
+
+## Prerequisite
+
+Install and enable the plugin. Afterwards, head into Streaming settings and enable "Allow remote control".
+The URL found in the "Remote control url" field is where the API can be reached. The auth used is Basic Auth.
+
+## Usage
+
+There is only one end-point and that is where a torrent stream can be requested.
+
+Both return the same responses and all responses are JSON encoded.
+All successfully authenticated responses have status code 200.
+
+## POST /streaming/stream
+
+POST body must be the raw torrent you want to stream. No form formatting or anything can be used.
+
+List of URL GET Arguments
+
+* **path**: Path inside the torrent file to either a folder or a file you want to stream. The plugin will try to guess the best one. **Optional**. **Default**: '' (i.e. find the best file in the whole torrent)
+* **infohash**: Infohash of the torrent you want to stream, can make it a bit faster as it can avoid reading POST body. **Optional**.
+* **label**: If label plugin is enabled and the torrent is actually added then give the torrent this label. **Optional**. **Default**: ''
+* **wait_for_end_pieces**: Wait for the first and last piece in the streamed file to be fully downloaded. Can be necessary for some video players. It also enforces that the torrent can be actually downloaded. If the key exist with any (even empty) value, the feature is enabled. **Optional**. **Default**: false
+
+## GET /streaming/stream
+
+* **infohash**: Does the same as when POSTed. **Mandatory**.
+* **path**: Does the same as when POSTed. **Optional**.
+* **wait_for_end_pieces**: Does the same as when POSTed. **Optional**.
+
+## Success Response
+
+```json
+{
+    "status": "success", # Always equals this
+    "filename" "horse.mkv", # Filename of the streamed torrent
+    "url": "http://example.com/" # URL where the file can be reached by e.g. a media player
+}
+```
+
+## Error Response
+
+```json
+{
+    "status": "error", # Always equals this
+    "message" "Torrent failed" # description for why it failed
+}
+```
 
 # Version Info
 
-## Version Unreleased
+## Version 0.10.5
+* Added support for serving files inline
+
+## Version 0.10.4
+* Trying to set max priority less as it destroys performance
+
+## Version 0.10.3
 * Added label support
 * Reverse proxy config / replace URL config
 * Ensure internal Deluge state is updated before trying to use it

setup.py

@@ -42,7 +42,7 @@ from setuptools import setup, find_packages
 __plugin_name__ = "Streaming"
 __author__ = "Anders Jensen"
 __author_email__ = "johndoee@tidalstream.org"
-__version__ = "0.10.3"
+__version__ = "0.10.5"
 __url__ = "https://github.com/JohnDoee/deluge-streaming"
 __license__ = "GPLv3"
 __description__ = "Enables streaming of files while downloading them."

streaming/core.py

@@ -71,6 +71,12 @@ AUDIO_STREAMABLE_EXTENSIONS = ['flac', 'mp3', 'oga']
 STREAMABLE_EXTENSIONS = set(VIDEO_STREAMABLE_EXTENSIONS + AUDIO_STREAMABLE_EXTENSIONS)
 TORRENT_CLEANUP_INTERVAL = timedelta(minutes=30)
 MAX_FILE_PRIORITY = 2
+MAX_PIECE_PRIORITY = 7
+MIN_WAIT_PIECE_PRIORITY_DELAY = timedelta(seconds=5)
+WITHIN_CHAIN_PERCENTAGE = 0.10
+MIN_PIECE_COUNT_FOR_CHAIN_CONSIDERATION = 40
+MIN_CHAIN_WAIT_DELAY = timedelta(seconds=8)
+
 
 DEFAULT_PREFS = {
     'ip': '127.0.0.1',
@@ -128,6 +134,7 @@ class Torrent(object):
         self.readers = {}
         self.cycle_lock = defer.DeferredLock()
         self.last_activity = datetime.now()
+        self.waited_pieces = set()
 
         self.torrent = get_torrent(infohash)
         status = self.torrent.get_status(['piece_length'])
@@ -161,11 +168,31 @@ class Torrent(object):
             last_available_piece = piece
 
         if last_available_piece is None:
-            logger.debug('Since we are waiting for a piece, setting priority for %s to max' % (needed_piece, ))
-            self.torrent.handle.set_piece_deadline(needed_piece, 0)
-            self.torrent.handle.piece_priority(needed_piece, 7)
+            logger.debug('Since we are waiting for a piece, we need to check if we should set piece %s to max' % (needed_piece, ))
 
+            is_next_in_chain = False
             f = self.get_file_from_offset(from_byte)
+            file_piece_count = (f['size'] // self.piece_length) + 1
+
+            if file_piece_count <= MIN_PIECE_COUNT_FOR_CHAIN_CONSIDERATION:
+                is_next_in_chain = True
+            else:
+                best_reader_from_byte = max(reader[1] for reader in self.readers.values() if reader[1] <= from_byte)
+                best_reader_piece = best_reader_from_byte // self.piece_length
+                downloading_pieces = self.get_currently_downloading()
+                # TODO: unfinished_piece can be None
+                for unfinished_piece, status in enumerate(self.torrent.status.pieces[best_reader_piece:], best_reader_piece):
+                    if not status and unfinished_piece not in downloading_pieces:
+                        break
+
+                piece_diff = best_reader_piece - unfinished_piece - 1
+                if unfinished_piece >= best_reader_piece or piece_diff / file_piece_count <= WITHIN_CHAIN_PERCENTAGE:
+                    is_next_in_chain = True
+
+            if not is_next_in_chain:
+                logger.debug('Not a next-in-chain piece, setting priority now')
+                self.torrent.handle.set_piece_deadline(needed_piece, 0)
+                self.torrent.handle.piece_priority(needed_piece, MAX_PIECE_PRIORITY)
 
             file_priorities = self.torrent.get_file_priorities()
             if file_priorities[f['index']] != MAX_FILE_PRIORITY:
@@ -173,11 +200,18 @@ class Torrent(object):
                 file_priorities[f['index']] = MAX_FILE_PRIORITY
                 self.torrent.set_file_priorities(file_priorities)
 
-            for _ in range(300):
+            for i in range(300):
                 if self.torrent.status.pieces[needed_piece]:
                     break
+
                 if not reactor.running:
                     return
+
+                if is_next_in_chain and i == MIN_CHAIN_WAIT_DELAY.total_seconds() * 5 and needed_piece not in self.get_currently_downloading():
+                    logger.debug('Next in chain waiting failed, setting priority')
+                    self.torrent.handle.set_piece_deadline(needed_piece, 0)
+                    self.torrent.handle.piece_priority(needed_piece, MAX_PIECE_PRIORITY)
+
                 time.sleep(0.2)
                 status = self.torrent.get_status(['pieces'])
 
@@ -263,6 +297,10 @@ class Torrent(object):
                 else:
                     file_ranges[path] = from_byte
 
+                reader_piece = from_byte // self.piece_length
+                self.torrent.handle.set_piece_deadline(reader_piece, 0)
+                self.torrent.handle.piece_priority(reader_piece, MAX_PIECE_PRIORITY)
+
                 for fileset_hash, fileset in self.filesets.items():
                     if path in fileset['files']:
                         if fileset_hash in fileset_ranges:
@@ -313,8 +351,6 @@ class Torrent(object):
 
                     if piece < current_piece:
                         self.torrent.handle.piece_priority(piece, 0)
-                    elif piece == current_piece:
-                        self.torrent.handle.piece_priority(piece, 7)
                     else:
                         self.torrent.handle.piece_priority(piece, 1)
 
@@ -523,9 +559,10 @@ class TorrentHandler(object):
                         wait_for_pieces.append(piece - 1)
 
             logger.debug('We want first and last piece first, these are the pieces: %r' % (wait_for_pieces, ))
-            for piece in wait_for_pieces:
-                torrent.handle.set_piece_deadline(piece, 0)
-                torrent.handle.piece_priority(piece, 7)
+            if wait_for_pieces:
+                for piece in wait_for_pieces:
+                    torrent.handle.set_piece_deadline(piece, 0)
+                    torrent.handle.piece_priority(piece, MAX_PIECE_PRIORITY)
 
             for _ in range(220):
                 status = torrent.get_status(['pieces'])
@@ -773,7 +810,7 @@ class Core(CorePluginBase):
 
     @export
     @defer.inlineCallbacks
-    def stream_torrent(self, infohash=None, url=None, filedump=None, filepath_or_index=None, includes_name=False, wait_for_end_pieces=False, label=None):
+    def stream_torrent(self, infohash=None, url=None, filedump=None, filepath_or_index=None, includes_name=False, wait_for_end_pieces=False, label=None, as_inline=False):
         logger.debug('Trying to stream infohash:%s, url:%s, filepath_or_index:%s' % (infohash, url, filepath_or_index))
         torrent = get_torrent(infohash)
 
@@ -815,7 +852,7 @@ class Core(CorePluginBase):
 
         try:
             stream_or_item = yield defer.maybeDeferred(self.torrent_handler.stream, infohash, fn, wait_for_end_pieces=wait_for_end_pieces)
-            stream_url = self.thomas_http_output.serve_item(stream_or_item)
+            stream_url = self.thomas_http_output.serve_item(stream_or_item, as_inline=as_inline)
         except:
             logger.exception('Failed to stream torrent')
             defer.returnValue({'status': 'error', 'message': 'failed to stream torrent'})

streaming/data/streaming.js

@@ -349,20 +349,23 @@ StreamingPlugin = Ext.extend(Deluge.Plugin, {
         deluge.preferences.addPage(this.prefsPage);
 
         console.log('Streaming plugin loaded');
-        var doStream = function (tid, fileIndex) {
-            deluge.client.streaming.stream_torrent(tid, null, null, fileIndex, true, {
+        var doStream = function (tid, fileIndex, asInline) {
+            deluge.client.streaming.stream_torrent(tid, null, null, fileIndex, true, false, null, asInline, {
                 success: function (result) {
-                    console.log('Got result', result);
                     if (result.status == 'success') {
-                        var url = result.url;
-                        if (result.use_stream_urls) {
-                            url = 'stream+' + url;
-                            if (result.auto_open_stream_urls) {
-                                window.location.assign(url);
-                                return;
+                        if (asInline) {
+                            window.open(result.url, '_blank');
+                        } else {
+                            var url = result.url;
+                            if (result.use_stream_urls) {
+                                url = 'stream+' + url;
+                                if (result.auto_open_stream_urls) {
+                                    window.location.assign(url);
+                                    return;
+                                }
                             }
+                            Ext.Msg.alert('Stream ready', 'URL for stream: <a target="_blank" href="' + url + '">' + url + '</a>');
                         }
-                        Ext.Msg.alert('Stream ready', 'URL for stream: <a target="_blank" href="' + url + '">' + url + '</a>');
                     } else {
                         Ext.Msg.alert('Stream failed', 'Error message: ' + result.message);
                     }
@@ -370,6 +373,28 @@ StreamingPlugin = Ext.extend(Deluge.Plugin, {
             })
         }
 
+        var triggerStreamFile = function (asInline) {
+            var files = deluge.details.items.items[2];
+            var nodes = files.getSelectionModel().getSelectedNodes();
+            if (nodes) {
+                var fileIndex = nodes[0].attributes.fileIndex;
+                var tid = files.torrentId;
+                if (fileIndex >= 0) {
+                    doStream(tid, fileIndex, asInline);
+                }
+            }
+        }
+
+        deluge.menus.filePriorities.addMenuItem({
+            id: 'playthis',
+            text: 'Play in browser',
+            iconCls: 'icon-resume',
+            handler: function (item, event) {
+                deluge.menus.filePriorities.hide();
+                triggerStreamFile(true);
+                return false;
+            }
+        });
 
         deluge.menus.filePriorities.addMenuItem({
             id: 'streamthis',
@@ -377,15 +402,26 @@ StreamingPlugin = Ext.extend(Deluge.Plugin, {
             iconCls: 'icon-down',
             handler: function (item, event) {
                 deluge.menus.filePriorities.hide();
-                var files = deluge.details.items.items[2];
-                var nodes = files.getSelectionModel().getSelectedNodes();
-                if (nodes) {
-                    var fileIndex = nodes[0].attributes.fileIndex;
-                    var tid = files.torrentId;
-                    if (fileIndex >= 0) {
-                        doStream(tid, fileIndex);
-                    }
-                }
+                triggerStreamFile(false);
+                return false;
+            }
+        });
+
+
+        var triggerStreamTorrent = function (asInline) {
+            var ids = deluge.torrents.getSelectedIds();
+            if (ids) {
+                doStream(ids[0], null, asInline);
+            }
+        }
+
+        deluge.menus.torrent.addMenuItem({
+            id: 'playthistorrent',
+            text: 'Play in browser',
+            iconCls: 'icon-resume',
+            handler: function (item, event) {
+                deluge.menus.torrent.hide();
+                triggerStreamTorrent(true);
                 return false;
             }
         });
@@ -396,10 +432,7 @@ StreamingPlugin = Ext.extend(Deluge.Plugin, {
             iconCls: 'icon-down',
             handler: function (item, event) {
                 deluge.menus.torrent.hide();
-                var ids = deluge.torrents.getSelectedIds();
-                if (ids) {
-                    doStream(ids[0]);
-                }
+                triggerStreamTorrent(false);
                 return false;
             }
         });