001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030/**
031 *
032 */
033package org.openimaj.video.xuggle;
034
035import java.io.File;
036import java.io.IOException;
037import java.io.InputStream;
038import java.net.MalformedURLException;
039import java.net.URI;
040import java.net.URISyntaxException;
041import java.net.URL;
042import java.util.concurrent.TimeUnit;
043
044import org.openimaj.audio.AudioFormat;
045import org.openimaj.audio.AudioStream;
046import org.openimaj.audio.SampleChunk;
047import org.openimaj.audio.timecode.AudioTimecode;
048
049import com.xuggle.mediatool.IMediaReader;
050import com.xuggle.mediatool.MediaToolAdapter;
051import com.xuggle.mediatool.ToolFactory;
052import com.xuggle.mediatool.event.IAudioSamplesEvent;
053import com.xuggle.xuggler.Global;
054import com.xuggle.xuggler.IAudioSamples;
055import com.xuggle.xuggler.ICodec;
056import com.xuggle.xuggler.IContainer;
057import com.xuggle.xuggler.IError;
058import com.xuggle.xuggler.IStream;
059import com.xuggle.xuggler.IStreamCoder;
060import com.xuggle.xuggler.io.URLProtocolManager;
061
062/**
063 * A wrapper for the Xuggle audio decoding system into the OpenIMAJ audio
064 * system.
065 * 
066 * @author David Dupplaw (dpd@ecs.soton.ac.uk)
067 * @created 8 Jun 2011
068 * 
069 */
070public class XuggleAudio extends AudioStream
071{
072        static {
073                URLProtocolManager.getManager().registerFactory("jar", new JarURLProtocolHandlerFactory());
074        }
075
076        /** The reader used to read the video */
077        private IMediaReader reader = null;
078
079        /** The stream index that we'll be reading from */
080        private int streamIndex = -1;
081
082        /** The current sample chunk - note this is reused */
083        private SampleChunk currentSamples = null;
084
085        /** Whether we've read a complete chunk */
086        private boolean chunkAvailable = false;
087
088        /** The timecode of the current sample chunk */
089        private final AudioTimecode currentTimecode = new AudioTimecode(0);
090
091        /** The length of the media */
092        private long length = -1;
093
094        /** The URL being read */
095        private final String url;
096
097        /** Whether to loop the file */
098        private final boolean loop;
099
100        /**
101         * Whether this class was constructed from a stream. Some functions are
102         * unavailable
103         */
104        private boolean constructedFromStream = false;
105
106        /**
107         * 
108         * 
109         * @author David Dupplaw (dpd@ecs.soton.ac.uk)
110         * @created 8 Jun 2011
111         * 
112         */
113        protected class ChunkGetter extends MediaToolAdapter
114        {
115                /**
116                 * {@inheritDoc}
117                 * 
118                 * @see com.xuggle.mediatool.MediaToolAdapter#onAudioSamples(com.xuggle.mediatool.event.IAudioSamplesEvent)
119                 */
120                @Override
121                public void onAudioSamples(final IAudioSamplesEvent event)
122                {
123                        // Get the samples
124                        final IAudioSamples aSamples = event.getAudioSamples();
125                        final byte[] rawBytes = aSamples.getData().
126                                        getByteArray(0, aSamples.getSize());
127                        XuggleAudio.this.currentSamples.setSamples(rawBytes);
128
129                        // Set the timecode of these samples
130                        // double timestampMillisecs =
131                        // rawBytes.length/format.getNumChannels() /
132                        // format.getSampleRateKHz();
133                        final long timestampMillisecs = TimeUnit.MILLISECONDS.convert(
134                                        event.getTimeStamp().longValue(), event.getTimeUnit());
135
136                        XuggleAudio.this.currentTimecode.setTimecodeInMilliseconds(
137                                        timestampMillisecs);
138
139                        XuggleAudio.this.currentSamples.setStartTimecode(
140                                        XuggleAudio.this.currentTimecode);
141
142                        XuggleAudio.this.currentSamples.getFormat().setNumChannels(
143                                        XuggleAudio.this.getFormat().getNumChannels());
144
145                        XuggleAudio.this.currentSamples.getFormat().setSigned(
146                                        XuggleAudio.this.getFormat().isSigned());
147
148                        XuggleAudio.this.currentSamples.getFormat().setBigEndian(
149                                        XuggleAudio.this.getFormat().isBigEndian());
150
151                        XuggleAudio.this.currentSamples.getFormat().setSampleRateKHz(
152                                        XuggleAudio.this.getFormat().getSampleRateKHz());
153
154                        XuggleAudio.this.chunkAvailable = true;
155                }
156        }
157
158        /**
159         * Default constructor that takes the file to read.
160         * 
161         * @param file
162         *            The file to read.
163         */
164        public XuggleAudio(final File file)
165        {
166                this(file.toURI().toString(), false);
167        }
168
169        /**
170         * Default constructor that takes the file to read.
171         * 
172         * @param file
173         *            The file to read.
174         * @param loop
175         *            Whether to loop indefinitely
176         */
177        public XuggleAudio(final File file, final boolean loop)
178        {
179                this(file.toURI().toString(), loop);
180        }
181
182        /**
183         * Default constructor that takes the location of a file to read. This can
184         * either be a filename or a URL.
185         * 
186         * @param u
187         *            The URL of the file to read
188         */
189        public XuggleAudio(final URL u)
190        {
191                this(u.toString(), false);
192        }
193
194        /**
195         * Default constructor that takes the location of a file to read. This can
196         * either be a filename or a URL.
197         * 
198         * @param u
199         *            The URL of the file to read
200         * @param loop
201         *            Whether to loop indefinitely
202         */
203        public XuggleAudio(final URL u, final boolean loop)
204        {
205                this(u.toString(), loop);
206        }
207
208        /**
209         * Default constructor that takes the location of a file to read. This can
210         * either be a filename or a URL.
211         * 
212         * @param url
213         *            The URL of the file to read
214         */
215        public XuggleAudio(final String url)
216        {
217                this(url, false);
218        }
219
220        /**
221         * Default constructor that takes the location of a file to read. This can
222         * either be a filename or a URL. The second parameter determines whether
223         * the file will loop indefinitely. If so, {@link #nextSampleChunk()} will
224         * never return null; otherwise this method will return null at the end of
225         * the video.
226         * 
227         * @param u
228         *            The URL of the file to read
229         * @param loop
230         *            Whether to loop indefinitely
231         */
232        public XuggleAudio(final String u, final boolean loop)
233        {
234                this.url = u;
235                this.loop = loop;
236                this.create(null);
237        }
238
239        /**
240         * Construct a xuggle audio object from the stream.
241         * 
242         * @param stream
243         *            The stream
244         */
245        public XuggleAudio(final InputStream stream)
246        {
247                this.url = "stream://local";
248                this.loop = false;
249                this.constructedFromStream = true;
250                this.create(stream);
251        }
252
253        /**
254         * Create the Xuggler reader
255         * 
256         * @param stream
257         *            Can be NULL; else the stream to create from.
258         */
259        private void create(final InputStream stream)
260        {
261                // If the reader is already open, we'll close it first and
262                // reinstantiate it.
263                if (this.reader != null && this.reader.isOpen())
264                {
265                        this.reader.close();
266                        this.reader = null;
267                }
268
269                // Check whether the string we have is a valid URI
270                IContainer container = null;
271                int openResult = 0;
272                try
273                {
274                        // Create the container to read our audio file
275                        container = IContainer.make();
276
277                        // If we have a stream, we'll create from the stream...
278                        if (stream != null)
279                        {
280                                openResult = container.open(stream, null, true, true);
281
282                                if (openResult < 0)
283                                        System.out.println("XuggleAudio could not open InputStream to audio.");
284                        }
285                        // otherwise we'll use the URL in the class
286                        else
287                        {
288                                final URI uri = new URI(this.url);
289
290                                // If it's a valid URI, we'll try to open the container using
291                                // the URI string.
292                                openResult = container.open(uri.toString(),
293                                                IContainer.Type.READ, null, true, true);
294
295                                // If there was an error trying to open the container in this
296                                // way,
297                                // it may be that we have a resource URL (which ffmpeg doesn't
298                                // understand), so we'll try opening an InputStream to the
299                                // resource.
300                                if (openResult < 0)
301                                {
302                                        System.out.println("URL " + this.url + " could not be opened by ffmpeg. " +
303                                                        "Trying to open a stream to the URL instead.");
304                                        final InputStream is = uri.toURL().openStream();
305                                        openResult = container.open(is, null, true, true);
306
307                                        if (openResult < 0)
308                                        {
309                                                System.out.println("Error opening container. Error " + openResult +
310                                                                " (" + IError.errorNumberToType(openResult).toString() + ")");
311                                                return;
312                                        }
313                                }
314                                else
315                                        System.out.println("Opened XuggleAudio stream ok: " + openResult);
316                        }
317                } catch (final URISyntaxException e2)
318                {
319                        e2.printStackTrace();
320                        return;
321                } catch (final MalformedURLException e)
322                {
323                        e.printStackTrace();
324                        return;
325                } catch (final IOException e)
326                {
327                        e.printStackTrace();
328                        return;
329                }
330
331                // Set up a new reader using the container that reads the images.
332                this.reader = ToolFactory.makeReader(container);
333                this.reader.addListener(new ChunkGetter());
334                this.reader.setCloseOnEofOnly(!this.loop);
335
336                // Find the audio stream.
337                IStream s = null;
338                int i = 0;
339                while (i < container.getNumStreams())
340                {
341                        s = container.getStream(i);
342                        if (s != null &&
343                                        s.getStreamCoder().getCodecType() == ICodec.Type.CODEC_TYPE_AUDIO)
344                        {
345                                // Save the stream index so that we only get frames from
346                                // this stream in the FrameGetter
347                                this.streamIndex = i;
348                                break;
349                        }
350                        i++;
351                }
352                System.out.println("Using audio stream " + this.streamIndex);
353
354                if (container.getDuration() == Global.NO_PTS)
355                        this.length = -1;
356                else
357                        this.length = (long) (s.getDuration() *
358                                        s.getTimeBase().getDouble() * 1000d);
359
360                // Get the coder for the audio stream
361                final IStreamCoder aAudioCoder = container.
362                                getStream(this.streamIndex).getStreamCoder();
363
364                System.out.println("Using stream code: " + aAudioCoder);
365
366                // Create an audio format object suitable for the audio
367                // samples from Xuggle files
368                final AudioFormat af = new AudioFormat(
369                                (int) IAudioSamples.findSampleBitDepth(aAudioCoder.getSampleFormat()),
370                                aAudioCoder.getSampleRate() / 1000d,
371                                aAudioCoder.getChannels());
372                af.setSigned(true);
373                af.setBigEndian(false);
374                super.format = af;
375
376                System.out.println("XuggleAudio using audio format: " + af);
377
378                this.currentSamples = new SampleChunk(af.clone());
379        }
380
381        protected int retries = 0;
382
383        /**
384         * {@inheritDoc}
385         * 
386         * @see org.openimaj.audio.AudioStream#nextSampleChunk()
387         */
388        @Override
389        public SampleChunk nextSampleChunk()
390        {
391                try
392                {
393                        IError e = null;
394                        while ((e = this.reader.readPacket()) == null && !this.chunkAvailable)
395                                ;
396
397                        if (!this.chunkAvailable || e != null && this.retries < 5)
398                        {
399                                this.reader.close();
400                                this.reader = null;
401                                if (e != null)
402                                {
403                                        System.err.println("Got audio demux error " + e.getDescription());
404                                        this.create(null);
405                                        this.retries++;
406                                }
407                                System.out.println("Closing audio stream " + this.url);
408                                return null;
409                        }
410
411                        this.chunkAvailable = false;
412                        return this.currentSamples;
413                } catch (final Exception e)
414                {
415                }
416
417                return null;
418        }
419
420        /**
421         * {@inheritDoc}
422         * 
423         * @see org.openimaj.audio.AudioStream#reset()
424         */
425        @Override
426        public void reset()
427        {
428                if (this.constructedFromStream)
429                {
430                        System.out.println("Cannot reset a stream of audio.");
431                        return;
432                }
433
434                if (this.reader == null || this.reader.getContainer() == null)
435                        this.create(null);
436                this.seek(0);
437        }
438
439        /**
440         * {@inheritDoc}
441         * 
442         * @see org.openimaj.audio.AudioStream#getLength()
443         */
444        @Override
445        public long getLength()
446        {
447                return this.length;
448        }
449
450        /**
451         * {@inheritDoc}
452         * 
453         * @see org.openimaj.audio.AudioStream#seek(long)
454         */
455        @Override
456        public void seek(final long timestamp)
457        {
458                if (this.constructedFromStream)
459                {
460                        System.out.println("Cannot seek within a stream of audio.");
461                        return;
462                }
463
464                if (this.reader == null || this.reader.getContainer() == null)
465                        this.create(null);
466
467                // Convert from milliseconds to stream timestamps
468                final double timebase = this.reader.getContainer().getStream(
469                                this.streamIndex).getTimeBase().getDouble();
470                final long position = (long) (timestamp / timebase);
471
472                final long min = Math.max(0, position - 100);
473                final long max = position;
474
475                // System.out.println( "Timebase: "+timebase+" of a second second");
476                // System.out.println( "Position to seek to (timebase units): "+position
477                // );
478                // System.out.println( "max: "+max+", min: "+min );
479
480                final int i = this.reader.getContainer().seekKeyFrame(this.streamIndex,
481                                min, position, max, 0);
482
483                // Check for errors
484                if (i < 0)
485                        System.err.println("Audio seek error (" + i + "): " + IError.errorNumberToType(i));
486                else
487                        this.nextSampleChunk();
488        }
489
490        /**
491         * Close the audio stream.
492         */
493        public synchronized void close()
494        {
495                if (this.reader != null)
496                {
497                        synchronized (this.reader)
498                        {
499                                if (this.reader.isOpen())
500                                {
501                                        this.reader.close();
502                                        this.reader = null;
503                                }
504                        }
505                }
506        }
507}