001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030/** 031 * 032 */ 033package org.openimaj.video.xuggle; 034 035import java.io.File; 036import java.io.IOException; 037import java.io.InputStream; 038import java.net.MalformedURLException; 039import java.net.URI; 040import java.net.URISyntaxException; 041import java.net.URL; 042import java.util.concurrent.TimeUnit; 043 044import org.openimaj.audio.AudioFormat; 045import org.openimaj.audio.AudioStream; 046import org.openimaj.audio.SampleChunk; 047import org.openimaj.audio.timecode.AudioTimecode; 048 049import com.xuggle.mediatool.IMediaReader; 050import com.xuggle.mediatool.MediaToolAdapter; 051import com.xuggle.mediatool.ToolFactory; 052import com.xuggle.mediatool.event.IAudioSamplesEvent; 053import com.xuggle.xuggler.Global; 054import com.xuggle.xuggler.IAudioSamples; 055import com.xuggle.xuggler.ICodec; 056import com.xuggle.xuggler.IContainer; 057import com.xuggle.xuggler.IError; 058import com.xuggle.xuggler.IStream; 059import com.xuggle.xuggler.IStreamCoder; 060import com.xuggle.xuggler.io.URLProtocolManager; 061 062/** 063 * A wrapper for the Xuggle audio decoding system into the OpenIMAJ audio 064 * system. 065 * 066 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 067 * @created 8 Jun 2011 068 * 069 */ 070public class XuggleAudio extends AudioStream 071{ 072 static { 073 URLProtocolManager.getManager().registerFactory("jar", new JarURLProtocolHandlerFactory()); 074 } 075 076 /** The reader used to read the video */ 077 private IMediaReader reader = null; 078 079 /** The stream index that we'll be reading from */ 080 private int streamIndex = -1; 081 082 /** The current sample chunk - note this is reused */ 083 private SampleChunk currentSamples = null; 084 085 /** Whether we've read a complete chunk */ 086 private boolean chunkAvailable = false; 087 088 /** The timecode of the current sample chunk */ 089 private final AudioTimecode currentTimecode = new AudioTimecode(0); 090 091 /** The length of the media */ 092 private long length = -1; 093 094 /** The URL being read */ 095 private final String url; 096 097 /** Whether to loop the file */ 098 private final boolean loop; 099 100 /** 101 * Whether this class was constructed from a stream. Some functions are 102 * unavailable 103 */ 104 private boolean constructedFromStream = false; 105 106 /** 107 * 108 * 109 * @author David Dupplaw (dpd@ecs.soton.ac.uk) 110 * @created 8 Jun 2011 111 * 112 */ 113 protected class ChunkGetter extends MediaToolAdapter 114 { 115 /** 116 * {@inheritDoc} 117 * 118 * @see com.xuggle.mediatool.MediaToolAdapter#onAudioSamples(com.xuggle.mediatool.event.IAudioSamplesEvent) 119 */ 120 @Override 121 public void onAudioSamples(final IAudioSamplesEvent event) 122 { 123 // Get the samples 124 final IAudioSamples aSamples = event.getAudioSamples(); 125 final byte[] rawBytes = aSamples.getData(). 126 getByteArray(0, aSamples.getSize()); 127 XuggleAudio.this.currentSamples.setSamples(rawBytes); 128 129 // Set the timecode of these samples 130 // double timestampMillisecs = 131 // rawBytes.length/format.getNumChannels() / 132 // format.getSampleRateKHz(); 133 final long timestampMillisecs = TimeUnit.MILLISECONDS.convert( 134 event.getTimeStamp().longValue(), event.getTimeUnit()); 135 136 XuggleAudio.this.currentTimecode.setTimecodeInMilliseconds( 137 timestampMillisecs); 138 139 XuggleAudio.this.currentSamples.setStartTimecode( 140 XuggleAudio.this.currentTimecode); 141 142 XuggleAudio.this.currentSamples.getFormat().setNumChannels( 143 XuggleAudio.this.getFormat().getNumChannels()); 144 145 XuggleAudio.this.currentSamples.getFormat().setSigned( 146 XuggleAudio.this.getFormat().isSigned()); 147 148 XuggleAudio.this.currentSamples.getFormat().setBigEndian( 149 XuggleAudio.this.getFormat().isBigEndian()); 150 151 XuggleAudio.this.currentSamples.getFormat().setSampleRateKHz( 152 XuggleAudio.this.getFormat().getSampleRateKHz()); 153 154 XuggleAudio.this.chunkAvailable = true; 155 } 156 } 157 158 /** 159 * Default constructor that takes the file to read. 160 * 161 * @param file 162 * The file to read. 163 */ 164 public XuggleAudio(final File file) 165 { 166 this(file.toURI().toString(), false); 167 } 168 169 /** 170 * Default constructor that takes the file to read. 171 * 172 * @param file 173 * The file to read. 174 * @param loop 175 * Whether to loop indefinitely 176 */ 177 public XuggleAudio(final File file, final boolean loop) 178 { 179 this(file.toURI().toString(), loop); 180 } 181 182 /** 183 * Default constructor that takes the location of a file to read. This can 184 * either be a filename or a URL. 185 * 186 * @param u 187 * The URL of the file to read 188 */ 189 public XuggleAudio(final URL u) 190 { 191 this(u.toString(), false); 192 } 193 194 /** 195 * Default constructor that takes the location of a file to read. This can 196 * either be a filename or a URL. 197 * 198 * @param u 199 * The URL of the file to read 200 * @param loop 201 * Whether to loop indefinitely 202 */ 203 public XuggleAudio(final URL u, final boolean loop) 204 { 205 this(u.toString(), loop); 206 } 207 208 /** 209 * Default constructor that takes the location of a file to read. This can 210 * either be a filename or a URL. 211 * 212 * @param url 213 * The URL of the file to read 214 */ 215 public XuggleAudio(final String url) 216 { 217 this(url, false); 218 } 219 220 /** 221 * Default constructor that takes the location of a file to read. This can 222 * either be a filename or a URL. The second parameter determines whether 223 * the file will loop indefinitely. If so, {@link #nextSampleChunk()} will 224 * never return null; otherwise this method will return null at the end of 225 * the video. 226 * 227 * @param u 228 * The URL of the file to read 229 * @param loop 230 * Whether to loop indefinitely 231 */ 232 public XuggleAudio(final String u, final boolean loop) 233 { 234 this.url = u; 235 this.loop = loop; 236 this.create(null); 237 } 238 239 /** 240 * Construct a xuggle audio object from the stream. 241 * 242 * @param stream 243 * The stream 244 */ 245 public XuggleAudio(final InputStream stream) 246 { 247 this.url = "stream://local"; 248 this.loop = false; 249 this.constructedFromStream = true; 250 this.create(stream); 251 } 252 253 /** 254 * Create the Xuggler reader 255 * 256 * @param stream 257 * Can be NULL; else the stream to create from. 258 */ 259 private void create(final InputStream stream) 260 { 261 // If the reader is already open, we'll close it first and 262 // reinstantiate it. 263 if (this.reader != null && this.reader.isOpen()) 264 { 265 this.reader.close(); 266 this.reader = null; 267 } 268 269 // Check whether the string we have is a valid URI 270 IContainer container = null; 271 int openResult = 0; 272 try 273 { 274 // Create the container to read our audio file 275 container = IContainer.make(); 276 277 // If we have a stream, we'll create from the stream... 278 if (stream != null) 279 { 280 openResult = container.open(stream, null, true, true); 281 282 if (openResult < 0) 283 System.out.println("XuggleAudio could not open InputStream to audio."); 284 } 285 // otherwise we'll use the URL in the class 286 else 287 { 288 final URI uri = new URI(this.url); 289 290 // If it's a valid URI, we'll try to open the container using 291 // the URI string. 292 openResult = container.open(uri.toString(), 293 IContainer.Type.READ, null, true, true); 294 295 // If there was an error trying to open the container in this 296 // way, 297 // it may be that we have a resource URL (which ffmpeg doesn't 298 // understand), so we'll try opening an InputStream to the 299 // resource. 300 if (openResult < 0) 301 { 302 System.out.println("URL " + this.url + " could not be opened by ffmpeg. " + 303 "Trying to open a stream to the URL instead."); 304 final InputStream is = uri.toURL().openStream(); 305 openResult = container.open(is, null, true, true); 306 307 if (openResult < 0) 308 { 309 System.out.println("Error opening container. Error " + openResult + 310 " (" + IError.errorNumberToType(openResult).toString() + ")"); 311 return; 312 } 313 } 314 else 315 System.out.println("Opened XuggleAudio stream ok: " + openResult); 316 } 317 } catch (final URISyntaxException e2) 318 { 319 e2.printStackTrace(); 320 return; 321 } catch (final MalformedURLException e) 322 { 323 e.printStackTrace(); 324 return; 325 } catch (final IOException e) 326 { 327 e.printStackTrace(); 328 return; 329 } 330 331 // Set up a new reader using the container that reads the images. 332 this.reader = ToolFactory.makeReader(container); 333 this.reader.addListener(new ChunkGetter()); 334 this.reader.setCloseOnEofOnly(!this.loop); 335 336 // Find the audio stream. 337 IStream s = null; 338 int i = 0; 339 while (i < container.getNumStreams()) 340 { 341 s = container.getStream(i); 342 if (s != null && 343 s.getStreamCoder().getCodecType() == ICodec.Type.CODEC_TYPE_AUDIO) 344 { 345 // Save the stream index so that we only get frames from 346 // this stream in the FrameGetter 347 this.streamIndex = i; 348 break; 349 } 350 i++; 351 } 352 System.out.println("Using audio stream " + this.streamIndex); 353 354 if (container.getDuration() == Global.NO_PTS) 355 this.length = -1; 356 else 357 this.length = (long) (s.getDuration() * 358 s.getTimeBase().getDouble() * 1000d); 359 360 // Get the coder for the audio stream 361 final IStreamCoder aAudioCoder = container. 362 getStream(this.streamIndex).getStreamCoder(); 363 364 System.out.println("Using stream code: " + aAudioCoder); 365 366 // Create an audio format object suitable for the audio 367 // samples from Xuggle files 368 final AudioFormat af = new AudioFormat( 369 (int) IAudioSamples.findSampleBitDepth(aAudioCoder.getSampleFormat()), 370 aAudioCoder.getSampleRate() / 1000d, 371 aAudioCoder.getChannels()); 372 af.setSigned(true); 373 af.setBigEndian(false); 374 super.format = af; 375 376 System.out.println("XuggleAudio using audio format: " + af); 377 378 this.currentSamples = new SampleChunk(af.clone()); 379 } 380 381 protected int retries = 0; 382 383 /** 384 * {@inheritDoc} 385 * 386 * @see org.openimaj.audio.AudioStream#nextSampleChunk() 387 */ 388 @Override 389 public SampleChunk nextSampleChunk() 390 { 391 try 392 { 393 IError e = null; 394 while ((e = this.reader.readPacket()) == null && !this.chunkAvailable) 395 ; 396 397 if (!this.chunkAvailable || e != null && this.retries < 5) 398 { 399 this.reader.close(); 400 this.reader = null; 401 if (e != null) 402 { 403 System.err.println("Got audio demux error " + e.getDescription()); 404 this.create(null); 405 this.retries++; 406 } 407 System.out.println("Closing audio stream " + this.url); 408 return null; 409 } 410 411 this.chunkAvailable = false; 412 return this.currentSamples; 413 } catch (final Exception e) 414 { 415 } 416 417 return null; 418 } 419 420 /** 421 * {@inheritDoc} 422 * 423 * @see org.openimaj.audio.AudioStream#reset() 424 */ 425 @Override 426 public void reset() 427 { 428 if (this.constructedFromStream) 429 { 430 System.out.println("Cannot reset a stream of audio."); 431 return; 432 } 433 434 if (this.reader == null || this.reader.getContainer() == null) 435 this.create(null); 436 this.seek(0); 437 } 438 439 /** 440 * {@inheritDoc} 441 * 442 * @see org.openimaj.audio.AudioStream#getLength() 443 */ 444 @Override 445 public long getLength() 446 { 447 return this.length; 448 } 449 450 /** 451 * {@inheritDoc} 452 * 453 * @see org.openimaj.audio.AudioStream#seek(long) 454 */ 455 @Override 456 public void seek(final long timestamp) 457 { 458 if (this.constructedFromStream) 459 { 460 System.out.println("Cannot seek within a stream of audio."); 461 return; 462 } 463 464 if (this.reader == null || this.reader.getContainer() == null) 465 this.create(null); 466 467 // Convert from milliseconds to stream timestamps 468 final double timebase = this.reader.getContainer().getStream( 469 this.streamIndex).getTimeBase().getDouble(); 470 final long position = (long) (timestamp / timebase); 471 472 final long min = Math.max(0, position - 100); 473 final long max = position; 474 475 // System.out.println( "Timebase: "+timebase+" of a second second"); 476 // System.out.println( "Position to seek to (timebase units): "+position 477 // ); 478 // System.out.println( "max: "+max+", min: "+min ); 479 480 final int i = this.reader.getContainer().seekKeyFrame(this.streamIndex, 481 min, position, max, 0); 482 483 // Check for errors 484 if (i < 0) 485 System.err.println("Audio seek error (" + i + "): " + IError.errorNumberToType(i)); 486 else 487 this.nextSampleChunk(); 488 } 489 490 /** 491 * Close the audio stream. 492 */ 493 public synchronized void close() 494 { 495 if (this.reader != null) 496 { 497 synchronized (this.reader) 498 { 499 if (this.reader.isOpen()) 500 { 501 this.reader.close(); 502 this.reader = null; 503 } 504 } 505 } 506 } 507}